def main():
ops, args = getopt.getopt(sys.argv[1:], 'b')
ops = dict(ops)
if '-b' in ops:
corpus = TaggedCorpusReader('tagged/', r'.*\.tagged')
tagged_sents_m = corpus.tagged_sents()
m0 = nltk.DefaultTagger('N')
m1 = nltk.UnigramTagger(tagged_sents_m, backoff=m0)
m2 = nltk.BigramTagger(tagged_sents_m, backoff=m1)
m3 = nltk.TrigramTagger(tagged_sents_m, backoff=m2)
os.makedirs(dir, exist_ok=True) # cria a diretoria
output_file = open(corpus_path, 'wb')
dump(m3, output_file, -1)
output_file.close()
else:
# load do corpus
corpus_input = open(corpus_path, 'rb')
tagger_corpus = load(corpus_input)
corpus_input.close()
# load do input
file_path = sys.argv[1]
file_input = open(file_path, 'r')
file_lines = file_input.readlines()
print("### LOAD DONE ###") # debug
triplos = []
for i in range(int(len(file_lines))):
if file_lines[i] != '\n': # process non empty lines
triplos = processLine(file_lines[i], tagger_corpus, triplos)
triplos.sort(key=sortTriplos)
# triplos = remTriplosLastN(3,triplos)
print(triplos)
nodes = get_nodes(triplos)
edgesW = triplos
draw(nodes, edgesW)
currentProgress(currentCity, totalCities, 'cities')
totalReviews = 0
for file in files:
corpusdir = corpus_path+file[0:-5]+'/'
if not os.path.isdir(corpusdir):
missingCorpus(corpusdir)
hotelcorpus = TaggedCorpusReader(corpusdir, '.*')
stopset = getDefaultStopset(set(hotelcorpus.words("stopset.txt")))
for review in hotelcorpus.fileids():
if review == "stopset.txt":
continue
content = hotelcorpus.tagged_sents(review)
if len(content) == 0:
continue
totalReviews += 1
trimmedTokens = []
for sentences in content:
for word, pos in sentences:
if word.isalpha() and word.lower() not in stopset:
trimmedTokens += [(word.lower(), pos)]
trans += [lemmatize.getLemmas(trimmedTokens)]
trimmedTokens = []
lemmatize.saveLemmaDict()
freqDict = cityFreq(city, corpus_path, files)
TotalHotels = freqDict['TotalHotels']
import nltk
from nltk.corpus import ConllChunkCorpusReader
from nltk.corpus.reader.tagged import TaggedCorpusReader
root = '/usr/local/share/nltk_data/corpora/MASC-for-NE/'
masc_for_ne = TaggedCorpusReader(root,'.*', '_')
sents = masc_for_ne.tagged_sents()
ne_sents = [nltk.ne_chunk(sent) for sent in sents]
root = "/usr/local/share/nltk_data/corpora/masc_conll/"
gold_corpus = ConllChunkCorpusReader(root,r".*\.conll", chunk_types=("DATE","PERSON","ORGANIZATION","LOCATION"))
gold_sents = gold_corpus.chunked_sents()