def prepare_pos_features(Language_model_set, output_file):
corpus_root = '/home1/c/cis530/data-hw2/' + Language_model_set
texts = PlaintextCorpusReader(corpus_root, '.*')
text = texts.words()
tagged_text = nltk.pos_tag(text)
merged_tag_text = mergeTags(tagged_text)
lists = seperate_pos(merged_tag_text)
nouns_dist = FreqDist(lists[0])
top_nouns = nouns_dist.keys()[:200]
verbs_dist = FreqDist(lists[1])
top_verbs =verbs_dist.keys()[:200]
advs_dist = FreqDist(lists[2])
top_advs =advs_dist.keys()[:100]
prep_dist = FreqDist(lists[3])
top_preps =prep_dist.keys()[:100]
adjs_dist = FreqDist(lists[4])
top_adjs =adjs_dist.keys()[:200]
out = open(output_file, 'w')
for n in top_nouns:
out.write('NN'+ n + '\n')
for v in top_verbs:
out.write('VV'+ v + '\n')
for av in top_advs:
out.write('ADV'+ av + '\n')
for p in top_preps:
out.write('PREP'+ p + '\n')
for aj in top_adjs:
out.write('ADJ'+ aj + '\n')
def fileids(self, years='*'): """ Returns list all files or files exist in specific folder(s) >>> len(hr.fileids()) 3206 >>> len(hr.fileids(years=1996)) 157 >>> len(hr.fileids(years=[1996,2007])) 246 >>> hr.fileids()[0] '1996/HAM2-960622.xml' """ if type(years) is int: years = [str(years)] if years=='*': wordlists = PlaintextCorpusReader(self.hamshahri_root, '.*\.xml') fids = wordlists.fileids() return fids else: fids = [] for year in years: wordlists = PlaintextCorpusReader(self.hamshahri_root, str(year) + '/.*\.xml') fids = fids + wordlists.fileids() return fids
def get_coarse_level_features(dataset, output_file):
# Import the corpus reader
corpus_root = '/home1/c/cis530/data-hw2/'+dataset
# Define the folder where the files are situated
files_dataset = PlaintextCorpusReader(corpus_root, '.*')
# Open the output_file
output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
# Read the stopwlist
stop_list = open('/home1/c/cis530/data-hw2/'+'stopwlist.txt').read()
types_stop_list=stop_list.split()
for fileid in files_dataset.fileids():
# Output the docid
output.write(dataset+'/'+fileid+' ')
# Output the topic_name
topic_name=fileid.split('/')[0]
output.write(topic_name+' ')
# Output the num_tokens
tokens=files_dataset.words(fileid)
output.write('tok:'+str(len(tokens))+' ')
# Output the num_types
types=set(tokens)
output.write('typ:'+str(len(types))+' ')
# Output the num_contents
output.write('con:'+str(len([w for w in tokens if w not in types_stop_list]))+' ')
# Output the num_sents
sents = files_dataset.sents(fileid)
output.write('sen:'+str(len(sents))+' ')
# Output the avg_slen
avg_slen=round(float(len(tokens))/float(len(sents)),2)
output.write('len:'+str(avg_slen)+' ')
# Output the num_caps
output.write('cap:'+str(len([w for w in tokens if w[0]>='A' and w[0]<='Z'])))
output.write('\n')
output.close()
def setData(domain):
# domain variable can take one of the following values
#
# "chicago_crime_data",
# "economics",
# "software_vulnerability",
# "cyber_threat",
# "articles",
# "msds"
corpus_root = getRoot(domain) # based on the selected domain corpus root will hold the relative address of the corpus
wordlists = PlaintextCorpusReader(corpus_root, '.*') # NLTK's laintextCorpusReader load text files in the root
words = wordlists.words() # and extract all the words in each file
my_stopwords = nltk.corpus.stopwords.words('english') # my_stopwords holds a list of non-relevant (stop) words in english
content = [w for w in words if w.lower() not in my_stopwords] # stop words are removed
content = [w for w in content if len(w) > 2] # words shorther than two(2) characters are removed
content = [w for w in content if not w.isdigit()] # digit only words (e.g. "10", "30", "450") are removed
result = {}
# a list of related words is created for each word in the content variable
for word in content:
result[word] = []
for sset in wn.synsets(word): # the first synonym of a set is selected, this can be expanded to the rest of the words in the set for more accuracy but at the cost of performace
for synset in sset.hyponyms(): # a set of hyponyms is added for the main synonym
result[word].append(synset.name[0:synset.name.find('.')])
return result,content # both the synonyms and the original word corpus is returned
def plot_cfreq(self,corpus,patt,n):
wordlists = PlaintextCorpusReader(corpus,patt)
fileids = wordlists.fileids()
for id in fileids:
words = wordlists.words(id)
fre = FreqDist(word.lower() for word in words if word.isalpha())
return fre.plot(n,cumulative=True)
def tokenisation (path):
tokens = []
min_length = 3
for dirs in os.walk(path):
corpus_root = dirs[0] #parcour l'arborescence du chemin
if corpus_root != path:
textlist = PlaintextCorpusReader(corpus_root,'.*')
for files in textlist.fileids():
test= corpus_root + '/' + files
fs = open(test,'r')
texte=fs.readlines()
texte=str(texte)
words = map(lambda word: word.lower(), wordpunct_tokenize(texte))
j=0
while j<len(words):
if words[j] not in cachedStopWords:
tokens.append(words[j] )
j+=1
fs.close()
p = re.compile('[a-zA-Z]+')
tokens_filtered = filter(lambda token: p.match(token) and len(token)>= min_length, tokens)
# vocab = []
# for words in tokens_filtered:
# vocab.append(SnowballStemmer("english").stem(words))
# tokens_filtered_sans = set(vocab)
tokens_filtered_sans = set(tokens_filtered)
tokens_filtered_sans = list(tokens_filtered_sans)
return tokens_filtered_sans
def similar (text, word):
if re.match ("^[a-zA-Z0-9_\(\),\.]+$", text) and re.match ("^[a-zA-Z0-9_]+$", word):
text = '%s.txt' % text
f = open(os.path.join(CORPUS_ROOT, text), 'r')
source = f.read()
f.close()
corpus = PlaintextCorpusReader(CORPUS_ROOT, [text])
n_text = nltk.text.Text(corpus.words(text))
context_index = nltk.text.ContextIndex(n_text.tokens, filter=lambda x:x.isalpha(), key=lambda s:s.lower())
word = word.lower()
wci = context_index._word_to_contexts
result = []
if word in wci.conditions():
contexts = set(wci[word])
fd = nltk.probability.FreqDist(w for w in wci.conditions() for c in wci[w] if c in contexts and not w == word)
words = nltk.util.tokenwrap(fd.keys()[:20])
for middle_word in words.split(' '):
for context in contexts:
if re.search ("/" + context[0] + "(\W|\s)+" + middle_word + "(\W|\s)+" + context[1] + "/i", source) != 'none':
print (context[0], middle_word, context[1])
result.append ({'word': word, 'context_left': context[0], 'context_right': context[1]})
return dumps ({'name': text, 'word': word, 'result': result})
def tokenize_report_sents(self, report_of_the_time):
re = ReportEnviroments()
new_corpus_reports_fileids_list = PlaintextCorpusReader(re.original_reports_corpus_path, '.*')
raw_text = new_corpus_reports_fileids_list.raw(report_of_the_time)
sentencas_raw = sent_tokenize(raw_text)
original_report_path = str(new_corpus_reports_fileids_list.abspath(report_of_the_time))
return sentencas_raw, original_report_path, report_of_the_time
def extractWordsOnly(self, article):
templist = []
listtextstring = []
articlename = article + '.txt'
#corpus_root = '/home/jesal/onedump/'
wl = PlaintextCorpusReader(corpus_root, '.*')
allwords = wl.words(fileids = articlename)
exturllist = self.extractexternalURL(article)
textstring = wl.raw(articlename)
for item in exturllist:
textstring = textstring.replace(item,' ')
#templist = re.sub(r'[.!,;?]', ' ', textstring).split()
templist = nltk.word_tokenize(textstring)
listtemp = []
for i in templist:
j = re.sub('[^A-Za-z]+', '', i)
listtemp.append(str(j))
templistfinal = []
templistfinal= self.removeEmpty(listtemp)
return templistfinal
def extractParasInList(name):
corpuslocation ='/Users/anis/seniorProject/aligned Paragraphs/algebra'
reader = PlaintextCorpusReader(corpuslocation, '.*\.txt')
# This gives the list of paragraphs. every paragraph list contains ist of sentences
# So it is a list of lists. Bunch of sentenses as a list joins together to make #lists of pararagraph
pList = []
paragraphlist = reader.paras(name) #'simpleTuring.txt'
numpara = len(paragraphlist)
for sentlist in paragraphlist:
#print sentlist
numsent = len(sentlist)
#print type(sentlist),
#print numsent
paraAsAList = []
# this loops through all the sentence lists and make them one list'''
for i in range(numsent):
paraAsAList = paraAsAList + sentlist[i]
#print paraAsAList # this is the whole parapragph as one list
paraAsAString = ""
for word in paraAsAList:
paraAsAString = paraAsAString + word + str(" ")
#print paraAsAString
pList.append(paraAsAString)
#print len(pList)
return pList
def extract_related_terms(self):
re = ReportEnviroments()
new_corpus_clusters_fileids_list = PlaintextCorpusReader(re.cluster_corpus_path, '.*')
raw_text_list = []
for i in range(len(new_corpus_clusters_fileids_list.fileids())):
raw_text_list.extend([[new_corpus_clusters_fileids_list.raw(fileids=new_corpus_clusters_fileids_list.fileids()[i])]])
return raw_text_list
def corpus_from_directory(path, filetype='.*'): ''' Make a corpus of all files in a given directory. Can limit type by passing the desired extension, proper format is, e.g., '.*\.txt' ''' corpus_reader = PlaintextCorpusReader(path, filetype) return nltk.Text( corpus_reader.words() )
def get_lm_features(dataset,output_file):
# Import the corpus reader
corpus_root = '/home1/c/cis530/data-hw2/'+dataset
# Define the folder where the files are situated
files_dataset = PlaintextCorpusReader(corpus_root, '.*')
fin_model = BigramModel('Finance',corpus_root)
hel_model = BigramModel('Health',corpus_root)
res_model = BigramModel('Computers_and_the_Internet',corpus_root)
co_model = BigramModel('Research',corpus_root)
output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
for fileid in files_dataset.fileids():
# Output the docid
output.write(dataset+'/'+fileid+' ')
# Output the topic_name
topic_name=fileid.split('/')[0]
output.write(topic_name+' ')
word_list = files_dataset.words(fileid)
finprob,finper = fin_model.get_prob_and_per(word_list)
hlprob,hlper = hel_model.get_prob_and_per(word_list)
resprob,resper = res_model.get_prob_and_per(word_list)
coprob,coper = co_model.get_prob_and_per(word_list)
output.write('finprob:'+str(round(finprob,1))+' ')
output.write('hlprob:'+str(round(hlprob,1))+' ')
output.write('resprob:'+str(round(resprob,1))+' ')
output.write('coprob:'+str(round(coprob,1))+' ')
output.write('finper:'+str(round(finper,1))+' ')
output.write('hlper:'+str(round(hlper,1))+' ')
output.write('resper:'+str(round(resper,1))+' ')
output.write('coper:'+str(round(coper,1))+' ')
output.write('\n')
output.close()
class Document(object):
"""
A container object for a set of chapters.
This allows us to keep track of document frequencies when computing them the
first time so we don't repeat computations for common words. It also handles
the PlaintextCorpusReader functions for us.
"""
def __init__(self, chapter_paths):
"""
Create a new Document.
chapter_paths - A list of the paths for chapters in the document.
"""
self.corpus = PlaintextCorpusReader("", chapter_paths)
self.chapter_lists = self._sanitize_chapters()
self.chapter_dists = [(FreqDist(chapter), chapter) for chapter in
self.chapter_lists]
self.words = {}
def get_chapters(self):
return self.chapter_lists
def average_chapter_frequency(self, word):
freqs = []
if word in self.words:
return self.words[word]
else:
for (dist, wordlist) in self.chapter_dists:
freqs.append(dist[word]/float(len(wordlist)))
# Store and return the average frequency
avg_frq = mean(freqs)
self.words[word] = avg_frq
return avg_frq
def _sanitize_chapters(self):
# Sanitize the wordlists and return them
lists = [self.corpus.words(file_id) for file_id in
self.corpus.fileids()]
new_lists = []
for word_list in lists:
# Convert everything to lowercase (e.g. so "the" and "The" match)
word_list = [word.lower() for word in word_list]
# Remove any punctuation
word_list = [re.sub('\p{P}','',word) for word in word_list]
# Remove stopwords, punctuation, and any empty word
stops = stopwords.words('english')
stops.append('')
stops.append('said')
word_list = [word for word in word_list if (word not in stops and
word.isalpha())]
new_lists.append(word_list)
return new_lists
def main():
# Corpus Location
#for training data
posTrainCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/pos_train'
negTrainCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/neg_train'
#for test data
posTestCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/pos_test'
negTestCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/neg_test'
# Create Plain Text Corpus for training data
posCorpus = PlaintextCorpusReader(posTrainCorpus, '.*')
negCorpus = PlaintextCorpusReader(negTrainCorpus, '.*')
# Create Plain Text Corpus for test data
posTstCorpus = PlaintextCorpusReader(posTestCorpus, '.*')
negTstCorpus = PlaintextCorpusReader(negTestCorpus, '.*')
#GetBigrams
posBigrams = nltk.bigrams(posCorpus.words())
negBigrams = nltk.bigrams(negCorpus.words())
#Get no. of words per corpus
posWordLen = len(posCorpus.words())
negWordLen = len(negCorpus.words())
# Creating object of Lang_Model_classifier
obj1 = Lang_Model_Classifier()
obj1.freq_dst(posCorpus, negCorpus)
#For negative test data
for filename in os.listdir(negTestCorpus):
wordSet = negTstCorpus.words(filename)
print '**Unigram**'
unigr = obj1.perp(wordSet)
print unigr
print '**Bigram**'
bigr = obj1.perpBi(nltk.bigrams(wordSet))
print bigr
#For positive test data
for filename in os.listdir(posTestCorpus):
wordSet2 = posTstCorpus.words(filename)
print '**Unigram**'
posunigr = obj1.perp(wordSet2)
print posunigr
print '**Bigram**'
posbigr = obj1.perpBi(nltk.bigrams(wordSet2))
print posbigr
def hybrid_cfdist():
sherlock_corpus = PlaintextCorpusReader(CORPUS_ROOT_SHERLOCK, '.*', encoding='utf-8')
sherlock_bigrams = nltk.bigrams(sherlock_corpus.words())
pokemon_corpus = PlaintextCorpusReader(CORPUS_ROOT_POKEMON, '.*', encoding='utf-8')
pokemon_bigrams = nltk.bigrams(pokemon_corpus.words())
return nltk.ConditionalFreqDist(sherlock_bigrams + pokemon_bigrams)
def main():
corpus_root = '../posts/'
newcorpus = PlaintextCorpusReader(corpus_root, '.*',
para_block_reader=read_block_no_metadata)
corpus_words = [w.lower() for w in newcorpus.words() if w.isalpha()]
corpus_sentences = newcorpus.sents()
analyst = TextAnalyst(corpus_words, corpus_sentences, 'french')
analyst.print_analyze()
def get_sub_directories(directory):
files = PlaintextCorpusReader(directory, ".*")
dirs = list()
for f in files.fileids():
if "/" in f:
if (f[:f.index("/")] not in dirs):
dirs.append(f[:f.index("/")])
return dirs
def raw_text_convertion(self, page1, page2, convertion_style):
if self.doc_ext == '.pdf':
system("pdftotext %s -enc UTF-8 -f %i -l %i %s.pdf %s.txt"
%(convertion_style, page1, page2, self.doc_dir, self.doc_dir))
raw_text = PlaintextCorpusReader(dirname(self.doc_dir), self.temp_text_doc).raw()
encoded_lowertext = raw_text.encode('utf-8').lower()
self.raw_text = re.sub(r'[0-9]', '', encoded_lowertext)
return self.raw_text
def get_corpus(corpusdir): newcorpus = PlaintextCorpusReader(corpusdir, '.*') titles = newcorpus.fileids() # returns all the .txt files in the dir words = [] for title in titles: newcorpus_txt = newcorpus.words(title) words.extend([ e for e in newcorpus_txt if re.match(r"[aA-zZ]",e)]) return words
def raw(): """ Returns raw text of corpus >>> raw()[:54] '# DELM' """ wordlists = PlaintextCorpusReader(bijankhan_root, bijankhan_fileid) return wordlists.raw(bijankhan_fileid)
def extractingFromFolders():
folder2 = os.path.expanduser('~\\My Documents\\Tara\\Ongoing\\CharacterCorpus\\Reference')
fileresult = os.path.expanduser('~\\My Documents\\Tara\\Ongoing\\CharacterCorpus\\results.txt')
refer = PlaintextCorpusReader(folder2, 'harrygrepster.txt')
grepster = refer.words()
results = open(fileresult, 'a')
completeWords = wordlist.words()
stoppers = stopwords.words()
return grepster, results, completeWords, stoppers
def GetTweets():
corpusdir = 'DB/'
newCorpus = PlaintextCorpusReader(corpusdir, '.*\.txt$') #Regex allows you to ignore .DS_Store
pattern = '\r\n' #Regex accepts \r\n as the next line encoding in each 'tweet' in the database
tweets = nltk.regexp_tokenize(newCorpus.raw(), pattern, gaps=True) #iterate through list, creating 'tweets'
tweets = [x.lower() for x in tweets] #make all strings lowercase to make matching easier
return tweets
def process_nan():
corpus_root = '../nan_samples/'
library = PlaintextCorpusReader(corpus_root, '.*', encoding='utf-8')
tokens = nltk.word_tokenize(library.raw())
tokens = map(lambda x: process_element(x), tokens)
nan_tokens=[]
for i in tokens:
nan_tokens+=i.split(' ')
return nan_tokens
def raw(self, fileid):
"""
Returns raw text of fileid
>>> hr.raw('1996/HAM2-960622.xml')[:38]
'<?xml version="1.0" encoding="UTF-8"?>'
"""
wordlists = PlaintextCorpusReader(self.hamshahri_root, fileid)
return wordlists.raw(fileid)
def read_BNC_baby_stem(root_local):
global fdist
BNC_baby = []
stemmer = SnowballStemmer("english")
wordlists = PlaintextCorpusReader(root_local, '.*', encoding='latin-1')
for word in wordlists.words():
BNC_baby.append(stemmer.stem(word))
fdist = FreqDist(word.lower() for word in BNC_baby)
return(fdist)
def compare(request):
errors = []
statistics=[]
stats=[]
for x in range(1,3):
cantoname = "canto"+str(x)+".txt"
w=PlaintextCorpusReader("./",cantoname);
w.words();
t=nltk.text.Text(w.words());
l_lines=len(line_tokenize(w.raw()))
l_uwords=len(set(w.words()))
l_words=len(w.words())
l_sents=len(w.sents())
l_paras=len(w.paras())
l_linperpara=l_lines/l_paras
statistics.append(x)
statistics.append("Number of Words - "+ str(l_words))
statistics.append("Number of Unique Words - "+ str(l_uwords))
statistics.append("Number of Setences - "+ str(l_sents))
statistics.append("Number of Lines - "+ str(l_lines))
statistics.append("Number of Paras - "+ str(l_paras))
statistics.append("Number of Lines/Paras - "+ str(l_linperpara))
lexical_density=l_words/l_uwords
l_wordpersent = l_words/l_sents
statistics.append("Lexical Density (Total/Uniq) words- "+ str(lexical_density))
statistics.append("Words per sentence - "+ str(l_wordpersent))
stats.append(statistics)
return render_to_response('compare.html', {'stats':statistics})
def stats(request):
errors = []
statistics=[]
if 'q' in request.GET:
q = request.GET['q']
if not q:
errors.append('Enter a Canto Number')
else:
cantoname = "canto"+q+".txt"
w=PlaintextCorpusReader("./",cantoname);
w.words();
t=nltk.text.Text(w.words());
l_lines=len(line_tokenize(w.raw()))
l_uwords=len(set(w.words()))
l_words=len(w.words())
l_sents=len(w.sents())
l_paras=len(w.paras())
l_linperpara=l_lines/l_paras
statistics.append("Number of Words - "+ str(l_words))
statistics.append("Number of Unique Words - "+ str(l_uwords))
statistics.append("Number of Setences - "+ str(l_sents))
statistics.append("Number of Lines - "+ str(l_lines))
statistics.append("Number of Paras - "+ str(l_paras))
statistics.append("Number of Lines/Paras - "+ str(l_linperpara))
lexical_density=l_words/l_uwords
l_wordpersent = l_words/l_sents
statistics.append("Lexical Density (Total/Uniq) words- "+ str(lexical_density))
statistics.append("Words per sentence - "+ str(l_wordpersent))
return render_to_response('stats.html', {'statistics':statistics})
return render_to_response('stats.html', {'errors': errors})
def big_event_sentences():
corpus_root = '../Brazil_NightClub_Fire/'
wordlists = PlaintextCorpusReader(corpus_root, '.*\.txt')
BigEvent = wordlists.raw()
sent_tokenizer = nltk.data.load('../nltkData/tokenizers/punkt/english.pickle')
BigEventSentences = sent_tokenizer.tokenize(BigEvent)
return BigEventSentences
def class_event_sentences():
corpus_root = '../Islip13Rain/'
wordlists = PlaintextCorpusReader(corpus_root, ".*\.txt")
ClassEvent = wordlists.raw()
sent_tokenizer = nltk.data.load('../nltkData/tokenizers/punkt/english.pickle')
ClassEventSentences = sent_tokenizer.tokenize(ClassEvent)
return ClassEventSentences
def dict_freq_mot_classe(classe, corpus):
dictio = {}
for dirs in os.walk(corpus):
corpus_root = dirs[0] #parcour l'arborescence du chemin
if corpus_root != corpus:
if os.path.basename(corpus_root) == classe:
textlist = PlaintextCorpusReader(corpus_root, '.*')
for files in textlist.fileids():
test = corpus_root + '/' + files
l = split(test)
d = freq_mot(l)
for mot in d:
if mot not in dictio:
dictio[mot] = 0
dictio[mot] = dictio[mot] + d[mot]
return dictio
def __init__(self, value, junk=junk_words, p=programming_skills):
self.value = value
self.junk_words = junk
self.programming = p
root = os.getcwd() + '/output/' + self.value
files = self.value + '.*'
text = PlaintextCorpusReader(root, files)
self.corpus = text
def prepData(setdir):
cor = []
senti = setdir[-3:]
files = ".*.txt"
corpustemp = PlaintextCorpusReader(setdir, files)
corpus = nltk.Text(corpustemp.words())
corpus = [x.lower() for x in corpus
] # eliminate counting same word twice due to case sensitivity
# stemming -> take root stem of words
stemmer = PorterStemmer()
stemmed_corpus = []
for x in corpus:
stemmed_corpus_elm = x
stemmed_corpus.append((stemmed_corpus_elm, senti))
return stemmed_corpus
def get_feature_set_tuples(feature_file):
tuple_list = []
path = feature_file.split('/')
file_name = path[(-1):]
root = ''
for i in range(len(path)-1):
root=root + path[i] + '/'
feature_file = PlaintextCorpusReader(root, file_name)
features = feature_file.words()
for each in features:
if(each[:2] == 'NN' or each[:2] =='VV'):
tuple_list.append((each[2:], each[:2]))
elif(each[:3] =='ADJ' or each[:3]=='ADV'):
tuple_list.append((each[3:], each[:3]))
elif(each[:4]=='PREP'):
tuple_list.append((each[4:], each[:4]))
return tuple_list
def Load_RawText_Collection(root):
raw_text_dict = {}
corpus_root = root
wordlists = PlaintextCorpusReader(corpus_root, '.*\.txt')
porter = nltk.PorterStemmer()
stopwords = nltk.corpus.stopwords.words('english')
for each_file in wordlists.fileids():
content_list = []
for each_sent in wordlists.sents(each_file):
sent_list = []
for w in each_sent:
if len(w) > 2 and w.isalpha() and w not in stopwords:
sent_list.append(porter.stem(w.lower()))
if len(sent_list) > 0:
content_list.append(each_sent)
raw_text_dict[each_file] = content_list
return raw_text_dict
def my_bar(self, corpus, patt, n):
wordlists = PlaintextCorpusReader(corpus, patt)
fileids = wordlists.fileids()
k = len(fileids)
fig1 = pylab.figure(1)
fig2 = pylab.figure(2)
fig3 = pylab.figure(3)
fig4 = pylab.figure(4)
for id in fileids:
i = fileids.index(id) + 1
words = wordlists.words(id)
fre = FreqDist(word.lower() for word in words if word.isalpha())
self.bar_per(fre, n, fig1, 2 * k, 2 * i, id)
self.bar_count(fre, n, fig2, 2 * k, 2 * i, id)
self.bar_freq(fre, n, fig3, 2 * k, 2 * i, id)
fig4 = self.plot_cfreq(corpus, patt, n)
pylab.show()
def getMarkovBatch():
last_word_sentences = defaultdict(list)
markov_lyrics.markov()
corpus_root = '/Users/divyasingh/Documents/MABLE/Story_telling_lyrics/words/corpus'
wordlists = PlaintextCorpusReader(corpus_root, '.*')
mega_sentences = (wordlists.sents('lyrics_batch.txt'))
if mega_sentences:
for sentence in mega_sentences:
lw = last_word(sentence)
last_word_sentences[ lw ].append(sentence)
keys = last_word_sentences.keys()
else:
getMarkovBatch()
# print "#############################################"
# print "#############################################"
# print keys, last_word_sentences
return keys, last_word_sentences
class TextCollection:
def __init__(self):
# Create a Corpus with all the data preprocessed with exercise 2 tokenizer
self.corpus = PlaintextCorpusReader(corpusdir, '.*/*', word_tokenizer=preprocess)
# Create the vectorial Space, creating each Vector
self.Text_vectors = []
for document in self.corpus.fileids():
self.Text_vectors.append(TextVector(document))
def get_coarse_level_features(dataset, output_file):
# accessing the corpus
corpus_root = '/home1/c/cis530/data-hw2/'
dataset_path = corpus_root + dataset
# Reading the files from the directories
files = PlaintextCorpusReader(dataset_path, '.*')
ids = files.fileids()
stopFile = PlaintextCorpusReader(corpus_root, 'stopwlist.txt')
stops = stopFile.words()
#Opening a file that has to be written to
out = open(output_file, 'w')
for i in range(0,len(ids) - 1):
#Initializing certain variables
tokens_count=0
types = 0
non_stops_count=0
sents_count = 0
avg_sent_len=0
cap_count = 0
tokens=files.words(ids[i])
#Computing Number of Tokens
tokens_count = len(tokens)
#Computing Number of types
types = len(set(tokens))
non_stops=[]
#Computing Number of Content Words
for t in tokens:
if t not in stops:
non_stops.append(t)
non_stops_count = len(non_stops)
#Finding Average Sentence Length
sent = []
sent = files.sents(ids[i])
sents_count = len(sent)
sent_len=0
for s in sent:
sent_len = sent_len + len(s)
avg_sent_len = sent_len/float(sents_count)
#Computing Number of Captilized Words
for c in non_stops:
if c.istitle():
cap_count = cap_count+1
current_file = dataset + '/' + ids[i]
e = current_file.split('/')
out.write(current_file +' '+ e[-2] + ' tok:' + str(tokens_count) + ' typ:' + \
str(types) + ' con:' + str(non_stops_count) + ' sen:' + str(sents_count) + ' len:' + str(avg_sent_len) + ' cap:' + str(cap_count)+ '\n')
out.flush()
def my_count(self, corpus, patt, n, filename):
wordlists = PlaintextCorpusReader(corpus, patt)
fileids = wordlists.fileids()
res = []
for id in fileids:
leng = len(wordlists.words(id))
wordc = len(set(wordlists.words(id)))
wor = "=> num corpus words: " + ` leng `
dis = "=> num distinct words: " + ` wordc `
ric = "=> ind lex richness: " + ` leng / wordc `
res.append(dis)
res.append(ric)
res.append(wor)
for word in sorted(set(wordlists.words(id))):
freq = (wordlists.words(id)).count(word)
f = "(" + word.lower() + "," + ` round(100 * (freq / leng),
1) ` + ")"
t = "(" + word.lower() + "," + ` freq ` + "/" + ` leng ` + ")"
res.append(f)
res.append(t)
out = open("/home/camilo/" + filename, "w")
try:
for t in res[:n]:
out.write(t + "\n")
finally:
out.close()
def test_corpus_reader():
본문 = """세종(世宗, 1397년 5월 7일 (음력 4월 10일) ~ 1450년 3월 30일(음력 2월 17일), 재위 : 1418년 ~ 1450년)은 조선의 제4대 국왕이며 언어학자이다. 그의 업적에 대한 존경의 의미를 담은 명칭인 세종대왕(世宗大王)으로 자주 일컬어진다.
성은 이(李), 휘는 도(祹), 본관은 전주(全州), 자는 원정(元正), 아명은 막동(莫同)이었다. 묘호는 세종(世宗)이며, 시호는 영문예무인성명효대왕(英文睿武仁聖明孝大王)이고, 명나라에서 받은 시호는 장헌(莊憲)이었다. 존시를 합치면 세종장헌영문예무인성명효대왕(世宗莊憲英文睿武仁聖明孝大王)이다."""
# 임시 파일 생성
with tempfile.NamedTemporaryFile(mode='w+', encoding='utf8') as fp:
fp.write(본문)
fp.seek(0)
파일경로 = fp.name
폴더 = os.path.dirname(파일경로)
파일명 = os.path.basename(파일경로)
# # 말뭉치 리더 생성
reader = PlaintextCorpusReader(root=폴더,
fileids=[파일명],
word_tokenizer=형태소_분석기)
# 형태소 분리 확인
분석결과 = reader.words()
assert 분석결과
def main():
options = get_options()
if not os.path.isdir(options.output):
os.makedirs(options.output)
wordlists = PlaintextCorpusReader(options.input, '.*\.txt$')
stopwords = list()
if options.stopwords:
stopwords = get_stopwords(options.stopwords)
for fileid in wordlists.fileids():
results = analyze_text(wordlists.words(fileid), fileid, stopwords,
options.min_length, options.min_freq,
options.total_ngrams, options.min_measure,
options.bigrams_only, options.trigrams_only)
write_results(results, os.path.join(options.output, fileid),
options.bigrams_only, options.trigrams_only)
def occStats(self,path,format,list,plotting):
wordlists = PlaintextCorpusReader(path,format)
fileids = wordlists.fileids()
k = len(fileids)
# computing rel frequencies
self.fileStats(path,fileids)
# plotting vars
figname = "GQs by class"
figpath = plotting +'/'+ figname.replace(' ', '-') + '-stats.pdf'
savpath = plotting +'/'+ figname.replace(' ', '-')
# plotting
MyPlot(self.stats,self.classstats,figname, "three",plotting,list) # per class (no regression)
# generating report
SaveStats(self.classstats,self.stats,figpath,savpath,plotting) # per class
class Song(TextCollection):
def __init__(self, fileid, title=None):
self.title = title
self.fileid = fileid
self.corpus = PlaintextCorpusReader('.',
fileid,
word_tokenizer=tokenizer)
super().__init__(self.corpus.words())
def __repr__(self):
title = self.corpus.sents()[0]
return "<Song " + ' '.join(title) + ">"
def __eq__(self, other):
return self.fileid == other.fileid
def __hash__(self):
return hash(self.fileid)
def load_corpus(self, corpus_root, regex='.*', encoding='utf8'):
'''
返回自己的语料库
corpus_root：语料所在的目录
regex：正则，默认导入目录下所有文件
'''
from nltk.corpus import PlaintextCorpusReader
wordlists = PlaintextCorpusReader(corpus_root, regex, encoding= encoding)
return wordlists
def create_model(self, corpus_name):
print('reading corpus')
reader = PlaintextCorpusReader(CORPUS_DIR, corpus_name)
print('padded everygram pipeline')
train_data, vocab = padded_everygram_pipeline(self.ngram,
(reader.sents()))
print('creando modelo')
# Crea modelo
model = MLE(self.ngram)
print('ajustando datos')
# Ajusta a los datos
model.fit(train_data, vocab)
print('ajusto')
return model
def occurrence_mot_i_corpus(mot, corpus):
compteur = 0
for dirs in os.walk(corpus):
corpus_root = dirs[0] #parcour l'arborescence du chemin
if corpus_root != corpus:
textlist = PlaintextCorpusReader(corpus_root, '.*')
for files in textlist.fileids():
test = corpus_root + '/' + files
x = open(test, 'r')
for ligne in x:
lign = ligne.split()
for mt in lign:
mt = mot_mot(alphabet, mt)
if mt == mot:
# if ligne.find(mot)>0:
compteur += 1
x.close()
return compteur
def occStats(self,path,format,list,plotting):
wordlists = PlaintextCorpusReader(path,format)
fileids = wordlists.fileids()
k = len(fileids)
# computing rel frequencies
self.fileStatsCum(path,fileids)
# plotting vars (2)
fignameCum = "Base GQs"
figpathCum = plotting +'/'+ fignameCum.replace(' ', '-') + '-stats.pdf'
savpathCum = plotting +'/'+ fignameCum.replace(' ', '-')
# plotting
MyPlot(self.statsCum,self.classstatsCum,fignameCum, "three",plotting,list) # cum
# generating report
SaveStats(self.classstatsCum,self.statsCum,figpathCum,savpathCum,plotting) # cum
def get_pos_features(dataset,feature_set_file,output_file):
# Import the corpus reader
corpus_root = '/home1/c/cis530/data-hw2/'+dataset
# Define the folder where the files are situated
files_dataset = PlaintextCorpusReader(corpus_root, '.*')
feature_list = open('/home1/c/cis530/data-hw2/'+feature_set_file).read().split()
output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
NOUNS=['NN','NNS','NP','NPS']
VERBS=['VB','VBD','VBG','VBN','VBP','VBZ']
ADJECTIVE=['JJ','JJR','JJS']
ADVERB=['RB','RBR','RBS']
PREPOSITION=['IN']
for fileid in files_dataset.fileids():
# Output the docid
output.write(dataset+'/'+fileid+' ')
# Output the topic_name
topic_name=fileid.split('/')[0]
output.write(topic_name+' ')
pos_tag_tuple = nltk.pos_tag(files_dataset.words(fileid))
# convert the tuple to list so that it can be changed
pos_tag_list=[list(pos_tag_tuple[i]) for i in range(len(pos_tag_tuple))]
# retag the POS and replace the word with tag_word
for w in pos_tag_list:
if w[1] in NOUNS:
w[0]='NN'+w[0]
w[1]='NN'
elif w[1] in VERBS:
w[0]='VV'+w[0]
w[1]='VV'
elif w[1] in ADJECTIVE:
w[0]='ADJ'+w[0]
w[1]='ADJ'
elif w[1] in ADVERB:
w[0]='ADV'+w[0]
w[1]='ADV'
elif w[1] in PREPOSITION:
w[0]='PREP'+w[0]
w[1]='PREP'
tw_list=[w[0] for w in pos_tag_list]
fd=FreqDist(tw_list)
for tw in feature_list:
output.write(tw+':'+str(fd[tw])+' ')
output.write('\n')
output.close()
def get_tfidf_featureset(
corpus_root="/Users/bryceanderson/Desktop/brosse/TwitterStuff",
idf_range=(.3,.7), rebuild=False):
if not rebuild and os.path.isfile("tfidf.pickle"):
print("* Returning pickled feature set *")
with open("tfidf.pickle","rb") as f:
return pickle.load(f)
DemCorpus = PlaintextCorpusReader("./Democrat",".*\.txt")
RepCorpus = PlaintextCorpusReader("./Republican",".*\.txt")
FullCorpus = PlaintextCorpusReader(".", "(Republican|Democrat)/.*\.txt")
print("processing dems...")
demStems = classifier.get_stems_dict(DemCorpus,True)
print("processing reps...")
repStems = classifier.get_stems_dict(RepCorpus,True)
print("computing IDFs...")
fullStems = demStems.copy()
fullStems.update(repStems)
stemSets = { f:set(sl) for f,sl in fullStems.items() }
listOfAllStems = list(set.union(*stemSets.values()))
idfs = pd.Series({ stem:
sum([ stem in stem_list for _,stem_list in stemSets.items() ])
for stem in listOfAllStems }) / len(FullCorpus.fileids())
idfs = idfs[idfs.between(*idf_range)]
#print(len(idfs))
#idfs.sort_values(ascending=False, inplace=True)
#idfspoints = idfs[0:100]
#idfspoints.plot(kind='bar', figsize=(30,30), fontsize=28)
print("computing TFs...")
featureset = []
for demStemDict in demStems.values():
featureset.append((compute_tfidf(demStemDict, idfs), 'D'))
for repStemDict in repStems.values():
featureset.append((compute_tfidf(repStemDict, idfs), 'R'))
print("* Pickling feature set *")
with open("tfidf.pickle","wb") as f:
pickle.dump((featureset,idfs),f)
return (featureset, idfs)
def get_CIQ_wordVector():
wordlists = PlaintextCorpusReader(root_path + "实验文件", fileids='.*\.tsv')
words = wordlists.words("CIQ_traindata_contents.tsv")
model = gensim.models.KeyedVectors.load_word2vec_format(
root_path + 'GoogleNews-vectors-negative300.bin', binary=True)
vocabulary = model.wv.vocab.keys()
file_object = open(root_path + "实验文件/CIQ_wordVector.tsv",
encoding='UTF-8',
mode='w')
file_object.write(f"{len(vocabulary)}\t{300}\n")
for w in words:
if len(w) != 20 and w in vocabulary:
vector_list = model.get_vector(w).tolist()
file_object.write(w + "\t")
for index in range(len(vector_list) - 1):
file_object.write(str(vector_list[index]) + ",")
file_object.write(str(vector_list[len(vector_list) - 1]) + "\n")
file_object.close()
def __init__(self, album_file):
"""
if the album info is given by `lyrics/kendrick/damn.json
then the corpus path is given by `lyrics/kendrick/damn/`
"""
# split into ('lyrics/kendrick/damn', '.json')
root, ext = os.path.splitext(album_file)
root += '/'
self.artist, self.title, song_titles = genius.parse_album_file(
album_file)
self.corpus = PlaintextCorpusReader(root,
r'.*\.txt',
word_tokenizer=tokenizer)
super().__init__(self.corpus.words())
self.tracks = [
Song(root + fileid, title=title)
for title, fileid in zip(song_titles, self.corpus.fileids())
]
def generate_doc_list(self):
print('generate_doc_list!')
all = Reader(self.path, '.*\.txt')
for id in all.fileids():
if re.match(FactFeel.file_path_expression, id):
if re.match(FactFeel.file_expression, id):
if re.match(FactFeel.fact_expression, id):
#print('fact :{}'.format(id))
self.facts.append(id)
elif re.match(FactFeel.feel_expression, id):
#print('feel :{}'.format(id))
self.feels.append(id)
('fs:{}'.format(all.fileids()))
print('feels{}'.format(self.feels))
print('facts{}'.format(self.facts))
self.doclist[FactFeel.FACT] = self.facts
self.doclist[FactFeel.FEEL] = self.feels
with open(self.doc_list_path, 'w+') as outfile:
json.dump(self.doclist, outfile)
def read_text(path):
if os.path.isdir(path):
raw = PlaintextCorpusReader(path, '.*').raw()
tokens = nltk.tokenize.word_tokenize(raw)
else:
f = open(path)
raw = f.read()
tokens = nltk.tokenize.word_tokenize(raw)
return Text(tokens)
def occurrence_mot_i_classe_stem(mot, classe, corpus):
compteur = 0
for dirs in os.walk(corpus):
corpus_root = dirs[0] #parcour l'arborescence du chemin
if os.path.basename(corpus_root) == classe:
textlist = PlaintextCorpusReader(corpus_root, '.*')
for files in textlist.fileids():
test = corpus_root + '/' + files
x = open(test, 'r')
for ligne in x:
lign = ligne.split()
for mt in lign:
mt = mot_mot(alphabet, mt)
mt = SnowballStemmer("english").stem(mt)
if mt == mot:
# if ligne.find(mot)>0:
compteur += 1
x.close()
return compteur
def extractParasInList(name):
corpuslocation ='/home/aniszaman/seniorProject/combined/carnivore'
reader = PlaintextCorpusReader(corpuslocation, '.*\.txt')
# This gives the list of paragraphs. every paragraph list contains ist of sentences
# So it is a list of lists. Bunch of sentenses as a list joins together to make #lists of pararagraph
pList = []
paragraphlist = reader.paras(name) #'simpleTuring.txt'
numpara = len(paragraphlist)
for sentlist in paragraphlist:
numsent = len(sentlist)
paraAsAList = []
# this loops through all the sentence lists and make them one list'''
for i in range(numsent):
paraAsAList = paraAsAList + sentlist[i]
paraAsAString = ""
for word in paraAsAList:
paraAsAString = paraAsAString + word + str(" ")
pList.append(paraAsAString)
return pList
def diccionario_bigramPalabras():
# Lectura y transformación de Corpus
corpus = PlaintextCorpusReader("Corpus", '.*')
tokenizer = RegexpTokenizer(r'[a-zA-Záéíóúñ]+')
tokens = tokenizer.tokenize(corpus.raw())
# Generación diccionario bigram de palabras + frecuencia
bigrams_orig = bigrams(tokens)
fdist = FreqDist(bigrams_orig)
dict_bigrams = {}
for b in fdist:
b_tr = (b[0], traducciones.traduce_numerico(b[1]))
try:
if dict_bigrams[b_tr][1] < fdist.get(b):
dict_bigrams[b_tr] = [b, fdist.get(b)]
except:
dict_bigrams[b_tr] = [b, fdist.get(b)]
return dict_bigrams
def get_sentences_for_text(corpus_root, filename, lang='english'):
"""Segments the given text into sentences.
Args:
corpus_root: Directory in which the text file is residing.
filename: Name of the text file.
lang: Tokenizer language. For possible values, look at:
${NLTK_DATA}/tokenizers/punkt
Returns:
Sentences in the given text.
"""
tokenizer_path = 'tokenizers/punkt/' + lang + '.pickle'
text = PlaintextCorpusReader(
corpus_root, [filename],
word_tokenizer=WhitespaceTokenizer(),
sent_tokenizer=nltk.data.LazyLoader(tokenizer_path))
return text.sents()
def textmetrics():
local = input(
u'Digite o caminho para a pasta onde está seu arquivo de texto :')
corpus_root = (str(local))
wordlists = PlaintextCorpusReader(corpus_root, '.*')
nomes = wordlists.fileids()
print("")
print("Arquivos de Texto Encontrados:")
print(nomes)
print("")
escolha = input("Escolha o arquivo que quer pesquisar :")
print("")
u = open(local + '\\' + escolha, 'rb').read().decode('latin-1')
t = nltk.word_tokenize(u)
tokens = len(t)
tokensunicos = len(set(t))
diver = tokens / tokensunicos
print(u"Número de tokens no texto: " + str(tokens))
print(u"Número de tokens ùnicos no texto: " + str(tokensunicos))
print(u"Diversidade Léxica: " + str(float(diver)))
palavras = []
for i in range(len(t)):
if t[i].isalpha():
trans = t[i].lower()
if len(trans) > 1 and trans not in stopwords.words('portuguese'):
palavras.append(trans)
fdist1 = FreqDist(palavras)
vocab = (fdist1.items())
vocab1 = sorted([(tpl[1], tpl) for tpl in vocab], reverse=True)
vocab2 = OrderedDict([tpl[1] for tpl in vocab1])
print('Lista das 50 palavras mais frequentes: ')
print(list(vocab2)[:50])
fdist1.plot(50, title=u'Distribuição de Frequencia (50 mais comuns):')
def __init__(self, files):
Corpus.corpus = PlaintextCorpusReader('./data/speeches', files)
Corpus.speech = Speech(self.corpus.raw(), self.corpus.words(),
self.corpus.sents(), self.corpus.paras(), None,
None, None, None)
Corpus.speeches = None
Corpus.years = [
int(year.split('.')[0]) for year in self.corpus.fileids()
]
Corpus.complementary = None
