# load_multiple_corpus_files('docs\\brown_hw\\Test\\') # TaggedCorpusReader for train set train_root = 'docs\\brown_hw_lowercase\\Train' train_reader = TaggedCorpusReader(train_root, '.*') train_words = train_reader.words() train_word_counts = nltk.FreqDist(train_words) # Words of train set that occurs only once train_word_counts_1 = keys_by_value(train_word_counts) # Words with tags of train set tagged_words_with_unk = [list(x) for x in train_reader.tagged_words()] # Makes words 'UNK' whose counts are one for index, tagged_word in enumerate(tagged_words_with_unk): if tagged_word[0] in train_word_counts_1.keys(): tagged_words_with_unk[index][0] = 'UNK' # Computes tag frequencies and put them into dictionary tag_frequency = {} for index, tagged_word in enumerate(tagged_words_with_unk): try: tag_frequency[tagged_word[1]] += 1 except KeyError: tag_frequency[tagged_word[1]] = 1
# Corpus texto simples from nltk.corpus import PlaintextCorpusReader loc = '/Users/rmoura/nltk_data/corpora/rai/textoSimples/' corpus1 = PlaintextCorpusReader(loc, '.*\.txt') print(corpus1.fileids()) print(corpus1.sents()) print(corpus1.words()) # Corpus texto etiquetado from nltk.corpus.reader.tagged import TaggedCorpusReader loc = '/Users/rmoura/nltk_data/corpora/rai/textoEtiquetas/' corpus2 = TaggedCorpusReader(loc, '.*\.txt') print(corpus2.fileids()) print(corpus2.words()) print("Palavras etiquetadas: ", corpus2.tagged_words()) print(corpus2.tagged_words('003.txt')) print("Sentencas diretas:") for s in corpus2.sents(): print(' '.join(s)) from nltk.corpus.reader import CategorizedPlaintextCorpusReader loc = '/Users/rmoura/nltk_data/corpora/rai/textoCategorias/' corpus3 = CategorizedPlaintextCorpusReader(loc, '.*\.txt', cat_file="categorias.txt") print(corpus3.fileids()) print(corpus3.categories()) print(corpus3.words(categories='brasnam')) # Definicao de stopwords stopwords = nltk.corpus.stopwords.words('portuguese') fd = nltk.FreqDist(w.lower() for w in corpus3.words())