from helpers.extensions import sort_dict_by_value, keys_by_value
# Lower cases the files and saves it to another location
# lower_files('docs\\brown_hw\\Train\\', 'docs\\brown_hw_lowercase\\Train\\')
# lower_files('docs\\brown_hw\\Test\\', 'docs\\brown_hw_lowercase\\Test\\')
# load_multiple_corpus_files('docs\\brown_hw\\Train\\')
# load_multiple_corpus_files('docs\\brown_hw\\Test\\')
# TaggedCorpusReader for train set
train_root = 'docs\\brown_hw_lowercase\\Train'
train_reader = TaggedCorpusReader(train_root, '.*')
train_words = train_reader.words()
train_word_counts = nltk.FreqDist(train_words)
# Words of train set that occurs only once
train_word_counts_1 = keys_by_value(train_word_counts)
# Words with tags of train set
tagged_words_with_unk = [list(x) for x in train_reader.tagged_words()]
# Makes words 'UNK' whose counts are one
for index, tagged_word in enumerate(tagged_words_with_unk):
if tagged_word[0] in train_word_counts_1.keys():
tagged_words_with_unk[index][0] = 'UNK'
# Computes tag frequencies and put them into dictionary
import nltk
# Corpus texto simples
from nltk.corpus import PlaintextCorpusReader
loc = '/Users/rmoura/nltk_data/corpora/rai/textoSimples/'
corpus1 = PlaintextCorpusReader(loc, '.*\.txt')
print(corpus1.fileids())
print(corpus1.sents())
print(corpus1.words())
# Corpus texto etiquetado
from nltk.corpus.reader.tagged import TaggedCorpusReader
loc = '/Users/rmoura/nltk_data/corpora/rai/textoEtiquetas/'
corpus2 = TaggedCorpusReader(loc, '.*\.txt')
print(corpus2.fileids())
print(corpus2.words())
print("Palavras etiquetadas: ", corpus2.tagged_words())
print(corpus2.tagged_words('003.txt'))
print("Sentencas diretas:")
for s in corpus2.sents():
print(' '.join(s))
from nltk.corpus.reader import CategorizedPlaintextCorpusReader
loc = '/Users/rmoura/nltk_data/corpora/rai/textoCategorias/'
corpus3 = CategorizedPlaintextCorpusReader(loc, '.*\.txt', cat_file="categorias.txt")
print(corpus3.fileids())
print(corpus3.categories())
print(corpus3.words(categories='brasnam'))
# Definicao de stopwords
stopwords = nltk.corpus.stopwords.words('portuguese')
files = cityFileSet[city]
currentCity += 1
trans = []
wordGroupDict = dict()
print '\n'+city+'\n'
currentProgress(currentCity, totalCities, 'cities')
totalReviews = 0
for file in files:
corpusdir = corpus_path+file[0:-5]+'/'
if not os.path.isdir(corpusdir):
missingCorpus(corpusdir)
hotelcorpus = TaggedCorpusReader(corpusdir, '.*')
stopset = getDefaultStopset(set(hotelcorpus.words("stopset.txt")))
for review in hotelcorpus.fileids():
if review == "stopset.txt":
continue
content = hotelcorpus.tagged_sents(review)
if len(content) == 0:
continue
totalReviews += 1
trimmedTokens = []
for sentences in content:
for word, pos in sentences:
if word.isalpha() and word.lower() not in stopset:
trimmedTokens += [(word.lower(), pos)]
trans += [lemmatize.getLemmas(trimmedTokens)]
trimmedTokens = []
:return:
'''
file_list = glob.glob(root_directory + "/**/*.txt", recursive=True)
return file_list
# Use the ANC tool to build an nltk version of the data here.
oanc_directory = root + "\\corpora\\oanc\\nltk-data\\travel_guides" # oanc/nltk-data"
oanc_files = build_file_list(oanc_directory)
# See http://www.nltk.org/howto/corpus.html
oanc_corpus = TaggedCorpusReader(
oanc_directory, oanc_files,
sep="_") # Specify that _ is used as a separator.
print(oanc_corpus.fileids())
x = oanc_corpus.words()[:50]
print(x)
y = oanc_corpus.paras()[:10]
"""
This script is an alternative/demo to scrapeWordHunt.py, but is not
used in this folder.
"""
SOURCE_NAME = "OANC-TRAV"
txt_file_name = "analogy_sentences_OANC-TRAV.txt"
csv_file_name = "analogy_names_OANC-TRAV.csv"
output_handler = open(root + "\\corpora\\extractions\\" + txt_file_name,
"w",
encoding="utf-8")
# Find the indices of all paragraphs that contain the patterns as listed in