def statsText(text, words):
fdist = FreqDist()
# formatted prints will work with Python2 and Python3
for word in word_tokenize(text):
fdist[word.lower()] += 1
#loop over the words in fdist and see if you can find those words in the wordslist keys. Since some words in the
#wordslist also has wildcard * at the end to denote anything after the initial word, we use Regex to match those
#rather than matching on equity; e.g wrong* will match wrong, wrongful, wrongfully, wronged etc...
frequencies = []
for word in words:
if '*' in word: #if word has * we need to compare it with each item in fdist...
wordRegEx = word.replace(
'*', '.*') #make it suitable for Regular Expression...
for k in fdist:
m = re.match(wordRegEx, k)
if m:
frequencies.append((word, fdist.freq(m.group())))
else:
frequencies.append((word, fdist.freq(word)))
return frequencies
def add_documents(self, document_entries):
"""
Add new documents to be indexed.
:param document_entries: a set of objects from the class DocumentEntry
"""
if document_entries:
forward = {
key: {(document_entries[key][0])}
for key in document_entries.keys()
}
for key in document_entries.keys():
freq_dist = FreqDist(document_entries[key][1])
for token in document_entries[key][1]:
if len(self.inverted_index) is 0 \
or self.__normalize(token) not in self.inverted_index.keys():
self.inverted_index[self.__normalize(token)] \
.add((freq_dist.get(token), key, freq_dist.freq(token)))
else:
if not freq_dist.get(token) is None:
self.inverted_index[self.__normalize(token)] \
.add((freq_dist.get(token), key, freq_dist.freq(token)))
self.forward_index.update(forward)
self.tf_idf()
self.dal.save(self.forward_index, 'forward_index.csv')
self.dal.save(self.inverted_index, 'inverted_index.csv')
def statsText(text, words):
fdist = FreqDist()
# formatted prints will work with Python2 and Python3
for word in word_tokenize(text):
fdist[word.lower()] += 1
#loop over the words in fdist and see if you can find those words in the wordslist keys. Since some words in the
#wordslist also has wildcard * at the end to denote anything after the initial word, we use Regex to match those
#rather than matching on equity; e.g wrong* will match wrong, wrongful, wrongfully, wronged etc...
frequencies = []
for word in words:
if '*' in word: #if word has * we need to compare it with each item in fdist...
wordRegEx = word.replace('*', '.*') #make it suitable for Regular Expression...
for k in fdist:
m = re.match(wordRegEx, k)
if m:
frequencies.append((word, fdist.freq(m.group())))
else:
frequencies.append((word, fdist.freq(word)))
return frequencies
def text_to_dict(docs, metric):
""" Create dictionaries of term frequencies based on documents
Metric must be either :attr:`FrequencyMetrics.TF` or :attr:`FrequencyMetrics.TF_IDF`.
"""
doc_freqs = FreqDist() # Distribution over how many documents each word appear in.
tf_dists = [] # List of TF distributions per document
# Create freq_dist for each document
for doc in docs:
doc = preprocess.preprocess_text(doc)
fd = FreqDist()
for word in doc: fd.inc(word)
doc_freqs.update(fd.samples())
tf_dists.append(fd)
num_docs = len(docs)
# Build dictionaries
dicts = []
for i, fd in enumerate(tf_dists):
if i%100==0: print ' dict',str(i)+'/'+str(len(tf_dists))
d = {}
if metric == FrequencyMetrics.TF:
for word in fd.samples():
d[word] = fd.freq(word)
elif metric == FrequencyMetrics.TF_IDF:
for word in fd.samples():
d[word] = fd.freq(word) * math.log(float(num_docs)/doc_freqs[word])
else:
raise ValueError("No such feature type: %s" % feature_type);
dicts.append(d)
return dicts
def text_to_vector(docs, metric):
""" Create frequency based feature-vector from text
Metric must be either :attr:`FrequencyMetrics.TF` or :attr:`FrequencyMetrics.TF_IDF`.
"""
doc_freqs = FreqDist() # Distribution over how many documents each word appear in.
tf_dists = [] # List of TF distributions per document
# Create freq_dist for each document
for doc in docs:
doc = preprocess.preprocess_text(doc)
fd = FreqDist()
for word in doc: fd.inc(word)
doc_freqs.update(fd.samples())
tf_dists.append(fd)
all_tokens = doc_freqs.keys()
num_docs = len(docs)
num_features = len(all_tokens)
# Build feature x document matrix
matrix = np.zeros((num_features, num_docs))
for i, fd in enumerate(tf_dists):
if metric == FrequencyMetrics.TF:
v = [fd.freq(word) for word in all_tokens]
elif metric == FrequencyMetrics.TF_IDF:
v = [fd.freq(word) * math.log(float(num_docs)/doc_freqs[word]) for word in all_tokens]
else:
raise ValueError("No such feature type: %s" % feature_type);
matrix[:,i] = v
return matrix
def get_ngram_features(tokens):
"""
This function creates the unigram and bigram features as described in
the assignment3 handout.
:param tokens:
:return: feature_vectors: a dictionary values for each ngram feature
"""
feature_vectors = {}
unigrams = ngrams(tokens, 1)
bigrams = ngrams(tokens, 2)
trigrams = ngrams(tokens, 3)
unigram_dist = FreqDist(word for word in unigrams)
bigram_dist = FreqDist(word for word in bigrams)
trigram_dist = FreqDist(word for word in trigrams)
for item in unigram_dist:
itemd = f'UNI_{item}'
feature_vectors[itemd] = unigram_dist.freq(item)
for item in bigram_dist:
itemd = f'BIGRAM_{item}'
feature_vectors[itemd] = bigram_dist.freq(item)
for item in trigram_dist:
itemd = f'TRIGRAM_{item}'
feature_vectors[itemd] = trigram_dist.freq(item)
return feature_vectors
def text_to_dict(docs, metric):
""" Create dictionaries of term frequencies based on documents
Metric must be either :attr:`FrequencyMetrics.TF` or :attr:`FrequencyMetrics.TF_IDF`.
"""
doc_freqs = FreqDist(
) # Distribution over how many documents each word appear in.
tf_dists = [] # List of TF distributions per document
# Create freq_dist for each document
for doc in docs:
doc = preprocess.preprocess_text(doc)
fd = FreqDist()
for word in doc:
fd.inc(word)
doc_freqs.update(fd.samples())
tf_dists.append(fd)
num_docs = len(docs)
# Build dictionaries
dicts = []
for i, fd in enumerate(tf_dists):
if i % 100 == 0: print ' dict', str(i) + '/' + str(len(tf_dists))
d = {}
if metric == FrequencyMetrics.TF:
for word in fd.samples():
d[word] = fd.freq(word)
elif metric == FrequencyMetrics.TF_IDF:
for word in fd.samples():
d[word] = fd.freq(word) * math.log(
float(num_docs) / doc_freqs[word])
else:
raise ValueError("No such feature type: %s" % feature_type)
dicts.append(d)
return dicts
def calculaEntropia(documento):
freq_dist = FreqDist()
corpus = Token(TEXT=open(documento).read())
WhitespaceTokenizer().tokenize(corpus)
for token in corpus['SUBTOKENS']:
freq_dist.inc(token['TEXT'])
entropia = 0
for i in freq_dist.samples():
entropia = entropia + (freq_dist.freq(i) * log(freq_dist.freq(i), 2))
return -entropia
def fun14():
"""counting other things"""
# print [len(w) for w in text1]
fdist1 = FreqDist([len(w) for w in text1])
# print fdist1.keys()
# print fdist1.items()
# word length 3 => 50223
print fdist1[3]
print fdist1.max()
# frequency 20%
print fdist1.freq(3)
def fun14():
"""counting other things"""
# print [len(w) for w in text1]
fdist1 = FreqDist([len(w) for w in text1])
# print fdist1.keys()
# print fdist1.items()
# word length 3 => 50223
print fdist1[3]
print fdist1.max()
# frequency 20%
print fdist1.freq(3)
def get_best_answers(self, passage_list, q):
logger = logging.getLogger("qa_logger")
logger.info("%s:\tAnswer Processing", q.id_q)
empty = passage_list == []
logger.info("%s:\t\tAnswer Extraction", q.id_q)
answer_list = []
for passage in passage_list:
a = passage.find_answer(q)
if a.is_successful():
answer_list.append(a)
if not answer_list:
return ([], empty)
logger.info("%s:\t\tAnswer Filtering", q.id_q)
# Obtain answer frequency
fd = FreqDist(answer_list)
# Normalize frequencies
normalize = fd.freq(fd.max())
# Modify scores by frequency
for answer in answer_list:
answer.score = int(answer.score * (fd.freq(answer) / normalize))
# Sort answers by score
answer_list.sort(key=lambda x: x.score, reverse=True)
# Filter bad answers
try:
threshold = int(MyConfig.get("answer_filtering", "threshold"))
except:
logger = logging.getLogger("qa_logger")
logger.error("answer quality threshold not found")
threshold = 50
answer_list = filter(lambda x: x.score > threshold, answer_list)
final_answers = []
for a in answer_list:
if a not in final_answers:
final_answers.append(a)
if len(final_answers) == 3:
break
return (final_answers, empty)
def get_best_answers(self, passage_list, q):
logger = logging.getLogger("qa_logger")
logger.info("%s:\tAnswer Processing", q.id_q)
empty = passage_list == []
logger.info("%s:\t\tAnswer Extraction", q.id_q)
answer_list = []
for passage in passage_list:
a = passage.find_answer(q)
if a.is_successful():
answer_list.append(a)
if not answer_list:
return ([], empty)
logger.info("%s:\t\tAnswer Filtering", q.id_q)
# Obtain answer frequency
fd = FreqDist(answer_list)
# Normalize frequencies
normalize = fd.freq(fd.max())
# Modify scores by frequency
for answer in answer_list:
answer.score = int(answer.score * (fd.freq(answer) / normalize))
# Sort answers by score
answer_list.sort(key=lambda x: x.score, reverse=True)
# Filter bad answers
try:
threshold = int(MyConfig.get("answer_filtering", "threshold"))
except:
logger = logging.getLogger("qa_logger")
logger.error("answer quality threshold not found")
threshold = 50
answer_list = filter(lambda x: x.score > threshold, answer_list)
final_answers = []
for a in answer_list:
if a not in final_answers:
final_answers.append(a)
if len(final_answers) == 3:
break
return (final_answers, empty)
class termBasedConsiderBackgroundModel(AbstractGenerativeModel):
def __init__(self, analyser, backgroundDistribution,
probOfBackgroundModel):
self.backGroundDistro = backgroundDistribution
self.analyser = analyser
self.ProbBackground = probOfBackgroundModel
def generateProbabilityDistribution(self, document_list):
tokens = []
for doc in document_list:
tokens += self.analyser(doc)
self.freqDist = FreqDist(tokens)
foreground_prob = 1 - self.ProbBackground
prob_distro = {}
backDistro = FreqDist()
for word in self.freqDist.keys():
backDistro[word] = self.backGroundDistro[word]
for word in self.freqDist.keys():
if word not in self.backGroundDistro.keys():
prob_distro[word] = (1.0 / foreground_prob) * (
self.freqDist.freq(word))
else:
prob_distro[word] = (1.0 / foreground_prob) * (
self.freqDist.freq(word) -
(self.ProbBackground * backDistro.freq(word)))
self.prob_distro = prob_distro
flag = True
for key in prob_distro:
if prob_distro[key] < 0 or prob_distro[key] > 1:
flag = False
break
return flag
def getProbabilityDistribution(self):
return self.prob_distro
def probOfDocument(self, document):
tokens = self.analyser(document)
prob = 1.0
for token in tokens:
if token in self.prob_distro:
prob *= self.prob_distro[token]
if prob == 1.0:
return 0.0
return prob
class termBased(AbstractGenerativeModel):
#It requires to pass analyser that will break document into tokens.
#It can also remove stopwords and make normalization for words.
def __init__(self, analyser):
self.analyser = analyser
print("Term Based ")
def generateProbabilityDistribution(self, document_list):
tokens = []
for doc in document_list:
tokens += self.analyser(doc)
self.freqDist = FreqDist(tokens)
def getProbabilityDistribution(self):
return self.freqDist
def probOfDocument(self, document):
tokens = self.analyser(document)
prob = 1.0
for token in tokens:
if self.freqDist.has_key(token):
prob *= self.freqDist.freq(token)
if prob == 1.0:
return 0.0
return prob
def generateProbabilityDistribution(self, document_list):
tokens = []
for doc in document_list:
tokens += self.analyser(doc)
self.freqDist = FreqDist(tokens)
foreground_prob = 1 - self.ProbBackground
prob_distro = {}
backDistro = FreqDist()
for word in self.freqDist.keys():
backDistro[word] = self.backGroundDistro[word]
for word in self.freqDist.keys():
if word not in self.backGroundDistro.keys():
prob_distro[word] = (1.0 / foreground_prob) * (
self.freqDist.freq(word))
else:
prob_distro[word] = (1.0 / foreground_prob) * (
self.freqDist.freq(word) -
(self.ProbBackground * backDistro.freq(word)))
self.prob_distro = prob_distro
flag = True
for key in prob_distro:
if prob_distro[key] < 0 or prob_distro[key] > 1:
flag = False
break
return flag
def statsText(text, words):
fdist = FreqDist()
# formatted prints will work with Python2 and Python3
for word in word_tokenize(text):
fdist.inc(word.lower())
return [(k, fdist.freq(k)) for k in words]
def statsText(text, words):
fdist = FreqDist()
# formatted prints will work with Python2 and Python3
for word in word_tokenize(text):
fdist.inc(word.lower())
return [(k, fdist.freq(k)) for k in words]
def zipfity(lst):
unigram = FreqDist()
for sent in lst:
for word in sent:
unigram[word.lower()] +=1 #the task didn't say anything so did it all to lower
sorted_unigram = sorted(unigram, key = unigram.get, reverse = True)
top10 = sorted_unigram[:10]
most_freq = unigram.freq(top10[0])
count = 1
print '{0:7s}{1:10s}{2:10s}'.format('word', 'obs.freq(%) ', 'zipf-law(%)')
print '----------------------------'
for word in top10:
print '{0:7s}{1:10.2f}{2:10.2f}'.format(word, unigram.freq(word)*100, (most_freq/count)*100)
count += 1
def get_opinion_features(tags):
"""
This function creates the opinion lexicon features
as described in the assignment3 handout.
the negative and positive data has been read into the following lists:
* neg_opinion
* pos_opinion
if you haven't downloaded the opinion lexicon, run the following commands:
* import nltk
* nltk.download('opinion_lexicon')
:param tags: tokens
:return: feature_vectors: a dictionary values for each opinion feature
"""
neg_opinion = opinion_lexicon.negative()
pos_opinion = opinion_lexicon.positive()
feature_vectors = {}
# YOUR CODE GOES HERE
feature_vectors.update({'UNI_POS_pretty': 0.01639344262295082})
feature_vectors.update({'UNI_POS_well': 0.013513513513513514})
feature_vectors.update({'UNI_POS_great': 0.023809523809523808})
feature_vectors.update({'UNI_POS_good': 0.03225806451612903})
feature_vectors.update({'UNI_POS_like': 0.016666666666666666})
feature_vectors.update({'UNI_NEG_unexpectedly': 0.0125})
feature_vectors.update({'UNI_POS_perfectly': 0.015151515151515152})
feature_vectors.update({'UNI_POS_thank': 0.016666666666666666})
feature_vectors.update({'UNI_POS_clearly': 0.013513513513513514})
feature_vectors.update({'UNI_NEG_confusing': 0.013513513513513514})
words = tags
wordF = FreqDist(words)
for word in neg_opinion:
if wordF.freq(word) > 0.0:
feature_vectors.update(
{'UNI_NEG_' + word: wordF[word] / len(wordF)})
for word in pos_opinion:
if wordF.freq(word) > 0.0:
feature_vectors.update(
{'UNI_POS_' + word: wordF[word] / len(wordF)})
return feature_vectors
def zipfity(lst):
unigram = FreqDist()
for sent in lst:
for word in sent:
unigram[word.lower(
)] += 1 #the task didn't say anything so did it all to lower
sorted_unigram = sorted(unigram, key=unigram.get, reverse=True)
top10 = sorted_unigram[:10]
most_freq = unigram.freq(top10[0])
count = 1
print '{0:7s}{1:10s}{2:10s}'.format('word', 'obs.freq(%) ', 'zipf-law(%)')
print '----------------------------'
for word in top10:
print '{0:7s}{1:10.2f}{2:10.2f}'.format(word,
unigram.freq(word) * 100,
(most_freq / count) * 100)
count += 1
def generate_weight_dictionary(self, service, words):
df = open(self.dictionary.get_dict_service_file_name(service), "w+")
t = Text(words)
freq_dist = FreqDist(t)
for w in freq_dist:
weight = 100 * freq_dist.freq(w)
df.write(w + helper.results_field_separator + str(weight) + "\n")
df.close()
def main():
argparser = argparse.ArgumentParser(description='text file')
argparser.add_argument('file', type=str, help='file to produce frequency distribution for')
args = argparser.parse_args()
#toker = WhitespaceTokenizer()
f = open(args.file)
text = f.read()
print(text)
fdist = FreqDist(text)
print(fdist.freq('28') * 100)
fdist.plot()
def text_to_vector(docs, metric):
""" Create frequency based feature-vector from text
Metric must be either :attr:`FrequencyMetrics.TF` or :attr:`FrequencyMetrics.TF_IDF`.
"""
doc_freqs = FreqDist(
) # Distribution over how many documents each word appear in.
tf_dists = [] # List of TF distributions per document
# Create freq_dist for each document
for doc in docs:
doc = preprocess.preprocess_text(doc)
fd = FreqDist()
for word in doc:
fd.inc(word)
doc_freqs.update(fd.samples())
tf_dists.append(fd)
all_tokens = doc_freqs.keys()
num_docs = len(docs)
num_features = len(all_tokens)
# Build feature x document matrix
matrix = np.zeros((num_features, num_docs))
for i, fd in enumerate(tf_dists):
if metric == FrequencyMetrics.TF:
v = [fd.freq(word) for word in all_tokens]
elif metric == FrequencyMetrics.TF_IDF:
v = [
fd.freq(word) * math.log(float(num_docs) / doc_freqs[word])
for word in all_tokens
]
else:
raise ValueError("No such feature type: %s" % feature_type)
matrix[:, i] = v
return matrix
def freq_lema_ngrams(list_monograms,list_lemas):
fdist1 = FreqDist(list_monograms)
#fdist2 = FreqDist(list_lemas)
vocabulary1 = fdist1.keys() #valores distintos
frec_grams=[];
for tag in vocabulary1:
temp1=[]
for i in range(len(list_monograms)):
if(list_monograms[i] == tag):
temp1.append(list_lemas[i])
temp2=set(temp1)
frec_grams.append([tag, fdist1[tag], fdist1.freq(tag),'-'.join(temp2)])
frec_grams_sort= sorted(frec_grams, key=itemgetter(1), reverse=True)
return frec_grams_sort
def main():
argparser = argparse.ArgumentParser(description='text file')
argparser.add_argument('file',
type=str,
help='file to produce frequency distribution for')
args = argparser.parse_args()
#toker = WhitespaceTokenizer()
f = open(args.file)
text = f.read()
print(text)
fdist = FreqDist(text)
print(fdist.freq('28') * 100)
fdist.plot()
def create_enhanced_dale_chall_list(self):
#list of sites used to create list of most frequent words
alexa_list = [
'Google', 'Facebook', 'YouTube', 'Yahoo!', 'Wikipedia',
'Microsoft', 'Amazon', 'Twitter', 'LinkedIn', 'Wordpress', 'Ebay',
'Apple', 'Paypal', 'Imdb', 'Tumblr', 'Disney', 'BBC', 'Livejasmin',
'Craigslist', 'Ask'
]
#bring all privacy texts into one list
corpus = []
data = get_all_policies()
for site in data:
if site in alexa_list:
corpus.append(data[site]["text"])
#get the words of this list into a list of words
t = textanalyzer("eng")
words = t.getWords("".join(corpus))
#open the dale chall wordlist
dale_chall_list = open(
'../nltk_contrib/dale_chall_wordlist.txt').read().split(';')
#create a text that consists of the words of the 20 privacy policies and delete all words that are on the dale-chall list of easy words
new_corpus = []
for word in words:
if word.lower() not in dale_chall_list and word not in alexa_list:
new_corpus.append(word.lower())
#create a frequency distribution of the words of this list of words
fdist = FreqDist(new_corpus)
#plot this
fdist.plot(80, cumulative=True)
#make a list of the words that make up 33% percent of the words that are not in the dale chall list (cummulative)
most_frequ = []
cum_percentage = 0.0
for sample in fdist:
cum_percentage += fdist.freq(sample)
most_frequ.append(sample)
if cum_percentage > 0.33:
break
#write those into a file
privacy_file = open("privacy_wordlist.txt", "w")
privacy_file.write(";".join(most_frequ))
def _entity_ranking(self, entities):
if len(entities) == 0:
return "", "", int(0)
# Obtain frequency of entities
entities_freq = FreqDist(entities)
# Our answer is the sample with the greatest number of outcomes
exact = entities_freq.max()
# Our window is empty because this algorithm generates exact answers
window = ""
# Our score is the entity frequency
score = int(entities_freq.freq(exact) * 1000)
return exact, window, score
def freq_lema_ngrams(list_monograms, list_lemas):
fdist1 = FreqDist(list_monograms)
#fdist2 = FreqDist(list_lemas)
vocabulary1 = fdist1.keys() #valores distintos
frec_grams = []
for tag in vocabulary1:
temp1 = []
for i in range(len(list_monograms)):
if (list_monograms[i] == tag):
temp1.append(list_lemas[i])
temp2 = set(temp1)
frec_grams.append(
[tag, fdist1[tag],
fdist1.freq(tag), '-'.join(temp2)])
frec_grams_sort = sorted(frec_grams, key=itemgetter(1), reverse=True)
return frec_grams_sort
def _entity_ranking(self, entities):
if len(entities) == 0:
return "", "", int(0)
# Obtain frequency of entities
entities_freq = FreqDist(entities)
# Our answer is the sample with the greatest number of outcomes
exact = entities_freq.max()
# Our window is empty because this algorithm generates exact answers
window = ""
# Our score is the entity frequency
score = int(entities_freq.freq(exact) * 1000)
return exact, window, score
def get_word_probs(sentences):
"""gets p of each word (freq / # of total tokens)"""
# make the corpus a non-nested list
corpus = []
for sentence in sentences:
corpus.extend(sentence)
# FreqDist does some of the heavy lifting
word_freq = FreqDist(corpus)
word_ps = {}
# store in a custom dict so we can update probabilities
for word in word_freq.keys():
word_ps[word] = word_freq.freq(word)
return (word_ps)
def create_enhanced_dale_chall_list(self):
#list of sites used to create list of most frequent words
alexa_list = ['Google', 'Facebook', 'YouTube', 'Yahoo!', 'Wikipedia', 'Microsoft', 'Amazon', 'Twitter', 'LinkedIn', 'Wordpress', 'Ebay', 'Apple', 'Paypal', 'Imdb', 'Tumblr', 'Disney', 'BBC', 'Livejasmin', 'Craigslist', 'Ask']
#bring all privacy texts into one list
corpus = []
data = get_all_policies()
for site in data:
if site in alexa_list:
corpus.append(data[site]["text"])
#get the words of this list into a list of words
t = textanalyzer("eng")
words = t.getWords("".join(corpus))
#open the dale chall wordlist
dale_chall_list = open('../nltk_contrib/dale_chall_wordlist.txt').read().split(';')
#create a text that consists of the words of the 20 privacy policies and delete all words that are on the dale-chall list of easy words
new_corpus = []
for word in words:
if word.lower() not in dale_chall_list and word not in alexa_list:
new_corpus.append(word.lower())
#create a frequency distribution of the words of this list of words
fdist = FreqDist(new_corpus)
#plot this
fdist.plot(80, cumulative=True)
#make a list of the words that make up 33% percent of the words that are not in the dale chall list (cummulative)
most_frequ = []
cum_percentage = 0.0
for sample in fdist:
cum_percentage += fdist.freq(sample)
most_frequ.append(sample)
if cum_percentage > 0.33:
break
#write those into a file
privacy_file = open("privacy_wordlist.txt", "w")
privacy_file.write(";".join(most_frequ))
def next(self, s, method = MOST_LIKELY):
# Pick a transition leaving state s and return a state that would
# likely follow. The next state is chosen according to the method
# specified. The default is to choose and return the most likely
# transition state.
# determine all states adjacent to s
transitions = self._adjacentVertices[s]
freqDist = FreqDist()
# determine the weights of the edges between state s and all adjacent states
for state in transitions:
freqDist.inc(state)
if method == MarkovChain.MOST_LIKELY:
return freqDist.max()
elif method == MarkovChain.LEAST_LIKELY:
# NLTK provides no built-in method to return the minimum of a
# frequency distribution so for now, we get a list of samples
# sorted in decreasing order and grab the last one.
return freqDist.sorted_samples()[-1]
else:
# choose a real number between 0 and 1
x = uniform(0,1)
# choose next state based on weights of the edges. Randomness plays a part here.
for i in range(len(transitions)):
probability = freqDist.freq(transitions[i])
if x < probability:
return transitions[i]
x = x - probability
exc = "Error in MarkovChain.next(). Did not find next state.\n"
raise exc
def main():
logger.info(f"starting now: {datetime.utcnow()}")
# read target and tokenize
with open(target, "r") as f:
tokens = word_tokenize(f.read())
logger.info(f"Tokenized {len(tokens)} words")
# filter english-only words
# most of should be cleaned in preprocessing but just in case
filtered = set(
t.lower() for t in tokens if
re.search(r"[\/+=<>0-9_:;,'@!()$|i\{\}\[\]?&*#%]", t.lower()) is None)
# no stopwords
less_stopwords = [
x.lower() for x in filtered if x not in stopwords.words("english")
]
logger.info(f"Filtered {len(filtered)} english only words")
# build a freq dist
fdist = FreqDist(w.lower() for w in tokens)
# sort by top most common words
common = sorted(
[(word, fdist.freq(word)) for word in less_stopwords],
key=lambda x: x[1],
reverse=True,
)[0:500]
# write to a file
with open(dest_csv, "w") as dest:
writer = csv.writer(dest)
for word in common:
writer.writerow([word[0], word[1]])
return 0
class FrequencySummarizer(base.BaseSummarizer):
"""
This class is based on [this](http://glowingpython.blogspot.com.co/2014/09/text-summarization-with-nltk.html) post
"""
stop_words = set()
sentences = list()
_frequency_distributions = None
_cleaned_text = list()
@property
def frequency_distributions(self):
return self._frequency_distributions
@frequency_distributions.setter
def frequency_distributions(self, frequency_distributions):
return
def summarize(self):
logger.debug("Extracting sentences")
self.sentences = sent_tokenize(self._text, language='spanish')
logger.debug("Extracting frequencies")
self._frequency_distributions = FreqDist(self._cleaned_text)
ranking = defaultdict(int)
for i, sentence in enumerate(self.sentences):
for word in sentence:
ranking[i] += self._frequency_distributions.freq(word)
ordered_sentences_by_priority = nlargest(
int(len(self.sentences) / 10) + 1, ranking, key=ranking.get)
return [self.sentences[i] for i in ordered_sentences_by_priority]
def __init__(self, text):
super().__init__(text)
self.stop_words = set(stopwords.words('spanish') + list(punctuation))
self._cleaned_text = [
x for x in word_tokenize(self.text, language='spanish')
if x not in self.stop_words
]
def get_content_avg_entropy(self):
'''
:return: avg entropy of text/<mime> parts for multipart bodies
'''
n = 0
txt_avg_ent = INIT_SCORE
# todo: make n-grams
tokens_list = tuple(self.pattern.get_stemmed_tokens())
#logger.debug(tokens_list)
for tokens in tokens_list:
#logger.debug(tokens)
n +=1
freqdist = FreqDist(tokens)
probs = [freqdist.freq(l) for l in FreqDist(tokens)]
txt_avg_ent += -sum([p * math.log(p,2) for p in probs])
#logger.debug(n)
# :))
if n !=0:
txt_avg_ent = txt_avg_ent/n
return txt_avg_ent
unigrams_path = reu_path + unigramsFrom
# count word length frequencies
for f in listdir(samples_path):
if (isfile(join(samples_path, f))):
output_path = reu_path + toDir + f
output = open(output_path, "w")
thisfile = open(samples_path + f).read()
tokens = tokenize(thisfile)
fd_words = FreqDist([len(w) for w in tokens])
for a in range(1, 21):
output.write(str(a) + '\t' + str(fd_words.freq(a)) + '\n')
count_20 = 0
# count 20+
for w in tokens:
if (len(w) >= 20):
count_20 += 1
output.write("20+\t" + str(count_20 / len(fd_words)) + '\n')
# count POS tag frequencies
for f in listdir(unigrams_path):
if (isfile(join(unigrams_path, f))):
output_path = reu_path + toDir + f
output = open(output_path, "a")
thisfile = open(unigrams_path + f).read()
tokens = tokenize(thisfile)
class MorphProbModel():
UNK_PROB = -99
def __init__(self,
beam=1000,
max_guess=20,
rare_treshold=10,
capitalization=True):
self._uni = FreqDist()
self._bi = ConditionalFreqDist()
self._tri = ConditionalFreqDist()
self._wd = ConditionalFreqDist()
self._l1 = 0.0
self._l2 = 0.0
self._l3 = 0.0
self._beam_size = beam
self._use_capitalization = capitalization
self._max_guess = max_guess
self._treshold = rare_treshold
self._unk = Guesser(10)
self._analyzer = None
self.cache = {}
def set_analyzer(self, obj):
self._analyzer = obj
def train(self, data):
C = False
for sent in data:
history = [('BOS', False), ('BOS', False)]
for w, l, t in sent:
# Ezt azért szedtem ki mert megeszik 4 giga memóriát ha marad
# t = encode((w, l, t))
if self._use_capitalization and w[0].isupper():
C = True
self._wd[w].inc(t)
self._uni.inc((t, C))
self._bi[history[1]].inc((t, C))
self._tri[tuple(history)].inc((t, C))
history.append((t, C))
history.pop(0)
C = False
for word, fd in self._wd.iteritems():
for tag, count in fd.iteritems():
if count < self._treshold:
self._unk.add_word(word.lower(), tag, count)
self._unk.finalize()
self._compute_lambda()
def _compute_lambda(self):
tl1 = 0.0
tl2 = 0.0
tl3 = 0.0
for history in self._tri.conditions():
(h1, h2) = history
for tag in self._tri[history].samples():
if self._uni[tag] == 1:
continue
c3 = self._safe_div((self._tri[history][tag] - 1),
(self._tri[history].N() - 1))
c2 = self._safe_div((self._bi[h2][tag] - 1),
(self._bi[h2].N() - 1))
c1 = self._safe_div((self._uni[tag] - 1), (self._uni.N() - 1))
if (c1 > c3) and (c1 > c2):
tl1 += self._tri[history][tag]
elif (c2 > c3) and (c2 > c1):
tl2 += self._tri[history][tag]
elif (c3 > c2) and (c3 > c1):
tl3 += self._tri[history][tag]
elif (c3 == c2) and (c3 > c1):
tl2 += float(self._tri[history][tag]) / 2.0
tl3 += float(self._tri[history][tag]) / 2.0
elif (c2 == c1) and (c1 > c3):
tl1 += float(self._tri[history][tag]) / 2.0
tl2 += float(self._tri[history][tag]) / 2.0
else:
pass
self._l1 = tl1 / (tl1 + tl2 + tl3)
self._l2 = tl2 / (tl1 + tl2 + tl3)
self._l3 = tl3 / (tl1 + tl2 + tl3)
def _safe_div(self, v1, v2):
if v2 == 0:
return -1
else:
return float(v1) / float(v2)
def _transition_prob(self, t, C, history):
p_uni = self._uni.freq((t, C))
p_bi = self._bi[history[-1]].freq((t, C))
p_tri = self._tri[tuple(history[-2:])].freq((t, C))
p = self._l1 * p_uni + self._l2 * p_bi + self._l3 * p_tri
if p == 0.0:
return self.UNK_PROB
return log(p, 2)
def _known_lexical_prob(self, word, t, C):
p = float(self._wd[word][t]) / float(self._uni[(t, C)])
return log(p, 2)
def _analyze(self, word):
tag_candidates = []
if word in self._wd.conditions():
tag_candidates = set(self._wd[word].samples())
else:
analyses = map(itemgetter(1), self._analyzer.analyze(word))
guesses = self._unk.get_probs(word.lower())
guesses = map(itemgetter(0),
sorted(guesses.iteritems(), reverse=True,
key=itemgetter(1))[:self._max_guess])
tag_candidates = set(guesses)
if analyses:
tag_candidates &= set(analyses)
if not tag_candidates:
tag_candidates = set(guesses)
return tag_candidates
def _lexical_prob(self, word, t, C):
if word in self._wd.conditions():
return self._known_lexical_prob(word, t, C)
else:
return self._unk.get_prob(word, t)
def tag(self, sent, n=5):
current_state = [(['BOS', 'BOS'], 0.0)]
out = self._tagword(sent, current_state, n)
return out
def _tagword(self, sent, current_states, n=5):
# A cache-sel elég gyors. Nem érdemes jobban vesződni vele.
if sent == []:
# yield ...
return [(map(itemgetter(0), tag_seq[0][2:]),
tag_seq[1]) for tag_seq in current_states[:n]]
word = sent[0]
sent = sent[1:]
new_states = []
# Cache lookup
sent_str = word + str(current_states)
if sent_str in self.cache:
return self._tagword(sent, self.cache[sent_str], n)
C = False
if self._use_capitalization and word[0].isupper():
C = True
analyses = self._analyze(word)
for (history, curr_sent_logprob) in current_states:
logprobs = []
for t in analyses:
p_t = self._transition_prob(t, C, history)
p_l = self._lexical_prob(word, t, C)
p = p_t + p_l
logprobs.append(((t, C), p))
for (tag, logprob) in logprobs:
new_states.append((history + [tag],
curr_sent_logprob + logprob))
new_states.sort(reverse=True, key=itemgetter(1))
if len(new_states) > self._beam_size:
new_states = new_states[:self._beam_size]
# Cache store
self.cache[sent_str] = new_states
# yield new_states
# self._tagword(sent, new_states, n)
return self._tagword(sent, new_states, n)
class TnT(TaggerI):
"""
TnT - Statistical POS tagger
IMPORTANT NOTES:
* DOES NOT AUTOMATICALLY DEAL WITH UNSEEN WORDS
- It is possible to provide an untrained POS tagger to
create tags for unknown words, see __init__ function
* SHOULD BE USED WITH SENTENCE-DELIMITED INPUT
- Due to the nature of this tagger, it works best when
trained over sentence delimited input.
- However it still produces good results if the training
data and testing data are separated on all punctuation eg: [,.?!]
- Input for training is expected to be a list of sentences
where each sentence is a list of (word, tag) tuples
- Input for tag function is a single sentence
Input for tagdata function is a list of sentences
Output is of a similar form
* Function provided to process text that is unsegmented
- Please see basic_sent_chop()
TnT uses a second order Markov model to produce tags for
a sequence of input, specifically:
argmax [Proj(P(t_i|t_i-1,t_i-2)P(w_i|t_i))] P(t_T+1 | t_T)
IE: the maximum projection of a set of probabilities
The set of possible tags for a given word is derived
from the training data. It is the set of all tags
that exact word has been assigned.
To speed up and get more precision, we can use log addition
to instead multiplication, specifically:
argmax [Sigma(log(P(t_i|t_i-1,t_i-2))+log(P(w_i|t_i)))] +
log(P(t_T+1|t_T))
The probability of a tag for a given word is the linear
interpolation of 3 markov models; a zero-order, first-order,
and a second order model.
P(t_i| t_i-1, t_i-2) = l1*P(t_i) + l2*P(t_i| t_i-1) +
l3*P(t_i| t_i-1, t_i-2)
A beam search is used to limit the memory usage of the algorithm.
The degree of the beam can be changed using N in the initialization.
N represents the maximum number of possible solutions to maintain
while tagging.
It is possible to differentiate the tags which are assigned to
capitalized words. However this does not result in a significant
gain in the accuracy of the results.
"""
def __init__(self, unk=None, Trained=False, N=1000, C=False):
"""
Construct a TnT statistical tagger. Tagger must be trained
before being used to tag input.
:param unk: instance of a POS tagger, conforms to TaggerI
:type unk: TaggerI
:param Trained: Indication that the POS tagger is trained or not
:type Trained: bool
:param N: Beam search degree (see above)
:type N: int
:param C: Capitalization flag
:type C: bool
Initializer, creates frequency distributions to be used
for tagging
_lx values represent the portion of the tri/bi/uni taggers
to be used to calculate the probability
N value is the number of possible solutions to maintain
while tagging. A good value for this is 1000
C is a boolean value which specifies to use or
not use the Capitalization of the word as additional
information for tagging.
NOTE: using capitalization may not increase the accuracy
of the tagger
"""
self._uni = FreqDist()
self._bi = ConditionalFreqDist()
self._tri = ConditionalFreqDist()
self._wd = ConditionalFreqDist()
self._eos = ConditionalFreqDist()
self._l1 = 0.0
self._l2 = 0.0
self._l3 = 0.0
self._N = N
self._C = C
self._T = Trained
self._unk = unk
# statistical tools (ignore or delete me)
self.unknown = 0
self.known = 0
def train(self, data):
"""
Uses a set of tagged data to train the tagger.
If an unknown word tagger is specified,
it is trained on the same data.
:param data: List of lists of (word, tag) tuples
:type data: tuple(str)
"""
# Ensure that local C flag is initialized before use
C = False
if self._unk is not None and self._T == False:
self._unk.train(data)
for sent in data:
history = [("BOS", False), ("BOS", False)]
for w, t in sent:
# if capitalization is requested,
# and the word begins with a capital
# set local flag C to True
if self._C and w[0].isupper():
C = True
self._wd[w][t] += 1
self._uni[(t, C)] += 1
self._bi[history[1]][(t, C)] += 1
self._tri[tuple(history)][(t, C)] += 1
history.append((t, C))
history.pop(0)
# set local flag C to false for the next word
C = False
self._eos[t]["EOS"] += 1
# compute lambda values from the trained frequency distributions
self._compute_lambda()
def _compute_lambda(self):
"""
creates lambda values based upon training data
NOTE: no need to explicitly reference C,
it is contained within the tag variable :: tag == (tag,C)
for each tag trigram (t1, t2, t3)
depending on the maximum value of
- f(t1,t2,t3)-1 / f(t1,t2)-1
- f(t2,t3)-1 / f(t2)-1
- f(t3)-1 / N-1
increment l3,l2, or l1 by f(t1,t2,t3)
ISSUES -- Resolutions:
if 2 values are equal, increment both lambda values
by (f(t1,t2,t3) / 2)
"""
# temporary lambda variables
tl1 = 0.0
tl2 = 0.0
tl3 = 0.0
# for each t1,t2 in system
for history in self._tri.conditions():
(h1, h2) = history
# for each t3 given t1,t2 in system
# (NOTE: tag actually represents (tag,C))
# However no effect within this function
for tag in self._tri[history].keys():
# if there has only been 1 occurrence of this tag in the data
# then ignore this trigram.
if self._uni[tag] == 1:
continue
# safe_div provides a safe floating point division
# it returns -1 if the denominator is 0
c3 = self._safe_div((self._tri[history][tag] - 1),
(self._tri[history].N() - 1))
c2 = self._safe_div((self._bi[h2][tag] - 1),
(self._bi[h2].N() - 1))
c1 = self._safe_div((self._uni[tag] - 1), (self._uni.N() - 1))
# if c1 is the maximum value:
if (c1 > c3) and (c1 > c2):
tl1 += self._tri[history][tag]
# if c2 is the maximum value
elif (c2 > c3) and (c2 > c1):
tl2 += self._tri[history][tag]
# if c3 is the maximum value
elif (c3 > c2) and (c3 > c1):
tl3 += self._tri[history][tag]
# if c3, and c2 are equal and larger than c1
elif (c3 == c2) and (c3 > c1):
tl2 += self._tri[history][tag] / 2.0
tl3 += self._tri[history][tag] / 2.0
# if c1, and c2 are equal and larger than c3
# this might be a dumb thing to do....(not sure yet)
elif (c2 == c1) and (c1 > c3):
tl1 += self._tri[history][tag] / 2.0
tl2 += self._tri[history][tag] / 2.0
# otherwise there might be a problem
# eg: all values = 0
else:
pass
# Lambda normalisation:
# ensures that l1+l2+l3 = 1
self._l1 = tl1 / (tl1 + tl2 + tl3)
self._l2 = tl2 / (tl1 + tl2 + tl3)
self._l3 = tl3 / (tl1 + tl2 + tl3)
def _safe_div(self, v1, v2):
"""
Safe floating point division function, does not allow division by 0
returns -1 if the denominator is 0
"""
if v2 == 0:
return -1
else:
return v1 / v2
def tagdata(self, data):
"""
Tags each sentence in a list of sentences
:param data:list of list of words
:type data: [[string,],]
:return: list of list of (word, tag) tuples
Invokes tag(sent) function for each sentence
compiles the results into a list of tagged sentences
each tagged sentence is a list of (word, tag) tuples
"""
res = []
for sent in data:
res1 = self.tag(sent)
res.append(res1)
return res
def tag(self, data):
"""
Tags a single sentence
:param data: list of words
:type data: [string,]
:return: [(word, tag),]
Calls recursive function '_tagword'
to produce a list of tags
Associates the sequence of returned tags
with the correct words in the input sequence
returns a list of (word, tag) tuples
"""
current_state = [(["BOS", "BOS"], 0.0)]
sent = list(data)
tags = self._tagword(sent, current_state)
res = []
for i in range(len(sent)):
# unpack and discard the C flags
(t, C) = tags[i + 2]
res.append((sent[i], t))
return res
def _tagword(self, sent, current_states):
"""
:param sent : List of words remaining in the sentence
:type sent : [word,]
:param current_states : List of possible tag combinations for
the sentence so far, and the log probability
associated with each tag combination
:type current_states : [([tag, ], logprob), ]
Tags the first word in the sentence and
recursively tags the reminder of sentence
Uses formula specified above to calculate the probability
of a particular tag
"""
# if this word marks the end of the sentence,
# return the most probable tag
if sent == []:
(h, logp) = current_states[0]
return h
# otherwise there are more words to be tagged
word = sent[0]
sent = sent[1:]
new_states = []
# if the Capitalisation is requested,
# initialise the flag for this word
C = False
if self._C and word[0].isupper():
C = True
# if word is known
# compute the set of possible tags
# and their associated log probabilities
if word in self._wd:
self.known += 1
for (history, curr_sent_logprob) in current_states:
logprobs = []
for t in self._wd[word].keys():
tC = (t, C)
p_uni = self._uni.freq(tC)
p_bi = self._bi[history[-1]].freq(tC)
p_tri = self._tri[tuple(history[-2:])].freq(tC)
p_wd = self._wd[word][t] / self._uni[tC]
p = self._l1 * p_uni + self._l2 * p_bi + self._l3 * p_tri
p2 = log(p, 2) + log(p_wd, 2)
# compute the result of appending each tag to this history
new_states.append((history + [tC], curr_sent_logprob + p2))
# otherwise a new word, set of possible tags is unknown
else:
self.unknown += 1
# since a set of possible tags,
# and the probability of each specific tag
# can not be returned from most classifiers:
# specify that any unknown words are tagged with certainty
p = 1
# if no unknown word tagger has been specified
# then use the tag 'Unk'
if self._unk is None:
tag = ("Unk", C)
# otherwise apply the unknown word tagger
else:
[(_w, t)] = list(self._unk.tag([word]))
tag = (t, C)
for (history, logprob) in current_states:
history.append(tag)
new_states = current_states
# now have computed a set of possible new_states
# sort states by log prob
# set is now ordered greatest to least log probability
new_states.sort(reverse=True, key=itemgetter(1))
# del everything after N (threshold)
# this is the beam search cut
if len(new_states) > self._N:
new_states = new_states[:self._N]
# compute the tags for the rest of the sentence
# return the best list of tags for the sentence
return self._tagword(sent, new_states)
sentences = sentences[:-1]
sentences = [s.strip() for s in sentences]
return sentences
#Answer to Question ii.a
#declare a counter for number of line
count = 1
#split the paragraph to sentences
sentences = split_into_sentences(text1)
#tokenize words for each line
for s in sentences:
lines = tokenizer.tokenize(s)
#print(lines)
#print word distribution
fdist2 = FreqDist(lines)
print("Probability of word [data] occuring in line " + str(count) + " is " + str((fdist2.freq('data') + fdist2.freq('Data'))))
#increment to next line number
count += 1
print("\n")
#Answer to Question ii.b
text2 = tokenizer.tokenize(text1.lower())
fdist3 = FreqDist(text2)
#print(fdist3)
print("The distribution of distinct word counts across the lines is as follows:")
for sample in fdist3:
print(sample + " " + str(fdist3[sample]))
print("\n")
def runTest(self,iteration):
print "running test %d"%iteration
pageUrl = '/reviews/www.zulily.com'
filename = '../test/resources/zulily.pkl'
try:
sjr = SiteJabberReviews(pageUrl,filename)
sjr.load()
helper = BayesHelper()
buckets = helper.generateLearningSetsFromReviews([sjr],[1,5],{'training': 0.8,'test':0.2})
self.assertEqual(len(buckets['training']), int(0.8*len(sjr.reviewsByRating[1])+int(0.8*len(sjr.reviewsByRating[5]))))
self.assertEqual(len(buckets['test']), int(0.2*len(sjr.reviewsByRating[1])+int(0.2*len(sjr.reviewsByRating[5]))))
# generate (term) tuples for FD -- this means we need to bust out like terms from combined distributions
allWords1 = [w for (textBag,rating) in buckets['training'] for w in textBag if rating == 1]
fd1 = FreqDist(allWords1)
allWords5 = [w for (textBag,rating) in buckets['training'] for w in textBag if rating == 5]
fd5 = FreqDist(allWords5)
commonTerms = [w for w in fd1.keys() if w in fd5.keys()]
commonTermFreqs = [(w,fd1.freq(w), fd5.freq(w), abs(fd1.freq(w) - fd5.freq(w))) for w in commonTerms]
commonTermFreqs.sort(key = itemgetter(3),reverse=True)
all_words = []
all_words.extend(allWords1)
all_words.extend(allWords5)
fdTrainingData = FreqDist(all_words)
# take an arbitrary subset of these
defaultWordSet = fdTrainingData.keys()[:2500]
#
def emitDefaultFeatures(tokenizedText):
'''
@param tokenizedText: an array of text features
@return: a feature map from that text.
'''
tokenizedTextSet = set(tokenizedText)
featureSet = {}
for text in defaultWordSet:
featureSet['contains:%s'%text] = text in tokenizedTextSet
return featureSet
classifier = None
encodedTrainSet = helper.encodeData(buckets['training'],emitDefaultFeatures )
classifier = nltk.NaiveBayesClassifier.train(encodedTrainSet)
encodedTestSet = helper.encodeData(buckets['test'], emitDefaultFeatures)
accuracy = nltk.classify.accuracy(classifier, encodedTestSet)
print "accuracy = %.9f"%accuracy
classifier.show_most_informative_features(10)
shouldBeClassed1 = []
shouldBeClassed5 = []
for (textbag, rating) in buckets['test']:
testRating = classifier.classify(emitDefaultFeatures(textbag))
if testRating != rating:
if rating == 1:
shouldBeClassed1.append(textbag)
else:
shouldBeClassed5.append(textbag)
print "length of mis-classified 1 star reviews = %d"%len(shouldBeClassed1)
print "length of mis-classified 5 star reviews = %d"%len(shouldBeClassed5)
print "length of all 1 star reviews submitted = %d"%len(sjr.reviewsByRating[1])
print "length of all 5 star reviews submitted = %d"%len(sjr.reviewsByRating[5])
print "length of test data for 1 star reviews = %d"%int(0.2*len(sjr.reviewsByRating[1]))
print "length of test data for 5 star reviews = %d"%int(0.2*len(sjr.reviewsByRating[5]))
return accuracy
except Exception as inst:
self.fail(inst)
# print "lines: ", len(lines)
for line in lines:
# print n, line.encode('utf-8')
line_tokens = tokenizer.tokenize(line)
# for token in line_tokens:
# print token.encode('utf-8'), " | "
# n = n + 1
text_array.append(line_tokens)
# now try to match hyphenated lines with their
# correpsonding beginning lines
n = 0
for line in text_array:
if len(line) > 0:
if line[-1][-1] == "-":
try:
line[-1] = line[-1][:-1] + text_array[n + 1][0]
text_array[n + 1] = text_array[n + 1][1:]
except IndexError as e:
print e
n = n + 1
# now flatten the 2d array
tokens = [item for sublist in text_array for item in sublist]
tokens = delete_non_greek_tokens(tokens)
for token in tokens:
fdist.inc(token)
print "most common: ", fdist.max().encode("utf-8")
for item in fdist.keys():
print item.encode("utf-8"), fdist.freq(item)
unigrams = regexp_tokenize(txt, pattern_unigrams)
bigrams = regexp_tokenize(txt, pattern_bigrams)
#Create frequency distributions
fdist_words = FreqDist(txt_tokens)
fdist_ngrams = FreqDist(unigrams + bigrams)
# Store most common words and ngrams for latter comparison of texts
words_most_common.append([k for (k,_) in fdist_words.most_common(params.n)])
ngrams_most_common.append([k for (k,_) in fdist_ngrams.most_common(params.m)])
outputname = "output_for_" + f.name.rsplit(os.sep, 2)[1]
# Write out the distribution of words in the document
with codecs.open("distributions-data/output/words_" + outputname, "w", encoding=my_encoding) as out:
for k,v in fdist_words.most_common():
prozent = fdist_words.freq(k)
out.write("{},{},{}\n".format(k,v, prozent))
# Write out the distribution of ngrams in the document
with codecs.open("distributions-data/output/letters_" + outputname, "w", encoding=my_encoding) as out:
for k,v in fdist_ngrams.most_common():
prozent = v / (len(unigrams) if len(k) == 1 else len(bigrams))
out.write("{},{},{}\n".format(k,v, prozent))
# Write the size of bins of words that appear with the same frequency
with codecs.open("distributions-data/bins/" + outputname, "w", encoding=my_encoding) as out:
for i in sorted(set(fdist_words.values())):
bin_size = fdist_words.Nr(i)
out.write("{},{}\n".format(i,bin_size))
print('Output distributions saved in \'output\' folder.')
print('Output bins saved in \'bins\' folder.')
# If there are many documents -> compare their most common words and ngrams
if len(params.files) > 1:
print(lexical_diversity(text3))
print(lexical_diversity(text5))
print(percentage(4, 5))
print(percentage(text4.count('a'), len(text4)))
# %%
fdist1 = FreqDist(text1)
fdist1
vocabulary1 = fdist1.keys()
print(vocabulary1)
print(fdist1['whale'])
# %%
fdist1.plot(50, cumulative=True)
# %%
list(fdist1.items())[0:5]
# %%
fdist1.freq('monstrous')
# %%
# Total number of samples
fdist1.N()
# %%
fdist1
# %%
print state_union_text.count("war")
state_union_text.concordance("economy")
state_union_text.similar("economy")
state_union_text.common_contexts(["economy", "jobs"])
from nltk.probability import FreqDist
fdist = FreqDist(state_union_text)
result = fdist.most_common(15)
result
from nltk.corpus import stopwords
stopwords.words("english")
filtered = [w for w in state_union.words() if not w in stopwords.words("english")]
len(filtered)
fdist_filtered = FreqDist(filtered)
fdist_filtered.most_common(20)
fdist_filtered.freq("good")/fdist_filtered.freq("bad")
fdist_filtered.freq("bad")/fdist_filtered.freq("evil")
fdist_filtered.plot(30)
def simulate_reading(net_text_input):
if len(net_text_input) > prm['max_text_len']:
raise ValueError(
'Text input {} has to be shorter than max_text_len: {}'.format(
net_text_input, prm['max_text_len']))
# Build the network.
nest.ResetKernel()
nest.SetKernelStatus({'local_num_threads': 9})
reset_reporting()
spike_groups.clear()
spike_decisions.clear()
nest.CopyModel('tsodyks2_synapse', 'head_grapheme_synapse_model',
prm['head_grapheme_synapse_model'])
nest.CopyModel('tsodyks2_synapse', 'letter_lexical_synapse_model',
prm['letter_lexical_synapse_model'])
local_vocabulary = [
w for w in vocabulary[unidecode(net_text_input[0])]
if distance_within(w, net_text_input, 4)
] # NOTE we may compare only stems to the full input!
graphemes_dist = FreqDist(
chain.from_iterable([decompose_word(w) for w in local_vocabulary]))
lexical_cols = dict([(w,
nest.Create(prm['neuron_type'],
prm['lexical_column_size']))
for w in local_vocabulary])
if prm['stems_and_suffixes']:
suffixes_cols = dict([(s,
nest.Create(prm['neuron_type'],
prm['lexical_column_size']))
for s in suffixes])
lexical_inhibiting_population = nest.Create(
prm['neuron_type'], prm['lexical_inhibiting_pop_size'])
letter_hypercolumns = [
make_hypercolumn(letters, prm['letter_column_size'])
for i in range(prm['max_text_len'])
]
# Reading heads' columns are sorted in separate lists by grapheme lengths.
reading_head_len_sorted = [
make_hypercolumn(size_graphemes, prm['head_column_size'])
for size_graphemes in graphemes_by_lengths
]
reading_head = {} # a 'flat' version
for len_graphemes in reading_head_len_sorted:
reading_head.update(len_graphemes)
grapheme_hypercolumns = [
make_hypercolumn(graphemes, prm['grapheme_column_size'])
for i in range(prm['max_text_len'])
]
# Make connections.
end_weight_dist = stats.norm(loc=len(net_text_input),
scale=3.0) # for exciting suffixes
for (hcol_n, hypercol) in enumerate(
letter_hypercolumns): # hypercol is: letter -> (neuron's nest id)
# Turn on appropriate letter columns.
if hcol_n < len(net_text_input) and net_text_input[hcol_n] in letters:
poisson_gen = nest.Create('poisson_generator', 1,
prm['letters_poisson_generator'])
nest.Connect(poisson_gen,
hypercol[net_text_input[hcol_n]],
syn_spec=prm['poisson_letter_excitation'])
###nest.SetStatus(hypercol[net_text_input[hcol_n]], prm['letter_neuron_params_on'])
# Letter hypercol's lateral inhibition to subsequent hypercols
for hypercol2 in letter_hypercolumns[hcol_n + 1:]:
nest.Connect(all_columns_cells(hypercol),
all_columns_cells(hypercol2),
syn_spec=prm['letter_col_lateral_inhibition'])
# Letter hypercol -> the reading head
for (letter, letter_col) in hypercol.items():
for (grapheme, grapheme_col) in reading_head.items():
if letter in grapheme:
nest.Connect(letter_col,
grapheme_col,
syn_spec=prm['letter_head_excitation'])
# Letter hypercol -> lexical units
for (word, word_col) in lexical_cols.items():
if hcol_n >= len(word):
nest.Connect(all_columns_cells(hypercol),
word_col,
syn_spec=prm['shorter_word_inhibition'])
else:
for (letter, letter_col) in hypercol.items():
if hcol_n == 0 and unidecode(
word[hcol_n]) == unidecode(letter):
nest.Connect(letter_col,
word_col,
syn_spec='letter_lexical_synapse_model')
nest.SetStatus(
nest.GetConnections(letter_col, word_col),
prm['member_first_letter_excitation'])
if (not prm['stems_and_suffixes']
and hcol_n == len(word) - 1 and unidecode(
word[len(word) - 1]) == unidecode(letter)):
nest.Connect(letter_col,
word_col,
syn_spec='letter_lexical_synapse_model')
nest.SetStatus(
nest.GetConnections(letter_col, word_col),
prm['member_last_letter_excitation'])
elif unidecode(letter) in unidecode(word):
nest.Connect(letter_col,
word_col,
syn_spec='letter_lexical_synapse_model')
nest.SetStatus(
nest.GetConnections(letter_col, word_col),
prm['member_letter_excitation'](len(word)))
else:
nest.Connect(letter_col,
word_col,
syn_spec='letter_lexical_synapse_model')
nest.SetStatus(
nest.GetConnections(letter_col, word_col),
prm['absent_letter_inhibition'](len(word)))
# Letter hypercol -> suffixes units
if prm['stems_and_suffixes']:
for (suffix, suffix_col) in suffixes_cols.items():
if len(net_text_input) - hcol_n <= len(suffix):
for (letter, letter_col) in hypercol.items():
if letter in suffix:
nest.Connect(
letter_col,
suffix_col,
syn_spec=prm['member_letter_excitation_suffix']
(len(suffix)))
else:
nest.Connect(
letter_col,
suffix_col,
syn_spec=prm['absent_letter_inhibition_suffix']
(len(suffix)))
for (grapheme, grapheme_col) in reading_head.items():
nest.Connect(
grapheme_col,
sum([
list(neurs) for hypercol in grapheme_hypercolumns
for (label, neurs) in hypercol.items() if label == grapheme
], []),
syn_spec='head_grapheme_synapse_model')
nest.Connect(all_columns_cells(lexical_cols),
lexical_inhibiting_population,
syn_spec=prm['lexical_inhibiting_pop_excitation'])
for (word, word_col) in lexical_cols.items():
nest.Connect(lexical_inhibiting_population,
word_col,
syn_spec=prm['lexical_inhibiting_pop_feedback'](
len(word)))
word_decomposition = decompose_word(word)
for (hcol_n, hypercol) in enumerate(grapheme_hypercolumns):
if hcol_n == len(word_decomposition):
break
nest.Connect(
word_col,
hypercol[word_decomposition[hcol_n]],
syn_spec={
'weight': (
prm['lexical_grapheme_base_excitation_weight']
# the excitation is stronger with rarer letters:
/ graphemes_dist.freq(word_decomposition[hcol_n]))
})
# Grapheme -> lexical feedback.
nest.Connect(hypercol[word_decomposition[hcol_n]],
word_col,
syn_spec=prm['grapheme_lexical_feedback'])
# Lateral inhibition for similar words.
for (word2, word2_col) in lexical_cols.items():
if word2 == word:
continue
elif distance_within(word, word2, 4):
nest.Connect(word_col,
word2_col,
syn_spec=prm['lexical_lateral_inhibition'])
if prm['stems_and_suffixes']:
for (suffix, suffix_col) in suffixes_cols.items():
# Lateral inhibition for suffixes.
for (suffix2, suffix2_col) in suffixes_cols.items():
if suffix != suffix2:
nest.Connect(suffix_col,
suffix2_col,
syn_spec=prm['suffix_lateral_inhibition'])
# Suffix -> grapheme connections.
for (hcol_n, hypercol) in enumerate(grapheme_hypercolumns):
# (weights will be assigned dynamically later)
nest.Connect(suffix_col,
all_columns_cells(hypercol),
syn_spec={'weight': 0.0})
for (hcol_n, hypercol) in enumerate(grapheme_hypercolumns):
for (grapheme, col) in hypercol.items():
# Lateral inhibition of graphemes containing at least one same letter
if hcol_n != 0:
for similar_grapheme in [
g for g in graphemes
if len(set(g).union(set(grapheme))) > 0
]:
nest.Connect(col,
grapheme_hypercolumns[hcol_n -
1][similar_grapheme],
syn_spec=prm['grapheme_lateral_inhibition'](
len(similar_grapheme)))
if hcol_n + 1 != prm['max_text_len']:
for similar_grapheme in [
g for g in graphemes
if len(set(g).union(set(grapheme))) > 0
]:
nest.Connect(col,
grapheme_hypercolumns[hcol_n +
1][similar_grapheme],
syn_spec=prm['grapheme_lateral_inhibition'](
len(similar_grapheme)))
# Insert probes:
for (word, word_col) in lexical_cols.items():
insert_probe(word_col, word, always_chart=False)
if prm['stems_and_suffixes']:
for (suffix, suffix_col) in suffixes_cols.items():
insert_probe(suffix_col, 'suff_' + suffix, always_chart=False)
insert_probe(lexical_inhibiting_population, 'lexical_inhibition')
##for (letter, letter_col) in letter_hypercolumns[1].items():
## insert_probe(letter_col, 'L2-'+letter)
for (grapheme, grapheme_col) in reading_head.items():
insert_probe(grapheme_col, 'head-' + grapheme, always_chart=False)
# [Reading facility config:]
spike_groups['Head'] = ['head-' + g for g in graphemes]
spike_groups['Words'] = local_vocabulary
if prm['stems_and_suffixes']:
spike_groups['Suffixes'] = ['suff_' + suff for suff in suffixes]
spike_decisions['Stems'] = [local_vocabulary]
spike_decisions['Reading'] = []
for (hcol_n, hypercol) in enumerate(grapheme_hypercolumns):
spike_decisions['Reading'].append([])
for (grapheme, grapheme_col) in hypercol.items():
insert_probe(grapheme_col,
'g{}-{}'.format(hcol_n, grapheme),
always_chart=False)
spike_decisions['Reading'][-1].append('g{}-{}'.format(
hcol_n, grapheme))
# Run the simulation, write readings.
nest.Simulate(prm['letter_focus_time'])
for step_n in range(prm['max_text_len']):
# Reassign the letter -> head weights (shifting skew normal).
weights_dist = stats.skewnorm(6, loc=step_n - 0.7, scale=0.67)
for assg_lett_n in range(prm['max_text_len']):
assg_hypercol = all_columns_cells(letter_hypercolumns[assg_lett_n])
for (ln, len_graphemes) in enumerate(reading_head_len_sorted):
len_graphemes = all_columns_cells(len_graphemes)
if len(len_graphemes) == 0:
continue
nest.SetStatus(
nest.GetConnections(assg_hypercol, len_graphemes), {
'weight':
(weights_dist.pdf(assg_lett_n) * 3000 /
(1.0 + (ln - 1) * prm['grapheme_length_damping']))
})
# Reassign the head -> grapheme weights (normal parametrized by time for each target hypercolumn).
for (hcol_n, hypercol) in enumerate(grapheme_hypercolumns):
weights_dist = stats.norm(
loc=hcol_n + 1,
scale=1.0) # hcol_n is treated as time step number
# (add one because of the first "dummy" step)
nest.SetStatus(
nest.GetConnections(all_columns_cells(reading_head),
all_columns_cells(hypercol)),
{
'weight':
weights_dist.pdf(1.0 + nest.GetKernelStatus('time') /
prm['letter_focus_time']) *
prm['head_grapheme_base_weight']
})
# Reassign the suffix -> grapheme weights (depending on estimated stem end).
if prm['stems_and_suffixes']: #### and step_n > len(net_text_input)/2:
stem_end = mean([
len(stem_reading[0]) for stem_reading in decide_spikes(
spike_decisions['Stems'])[:15]
])
#print(stem_end)
for (suffix, suffix_col) in suffixes_cols.items():
suffix_decomposition = decompose_word(suffix)
for grapheme in set(suffix_decomposition):
# Each occurence of a grapheme in suffix must exert is
# influence individually, they are then summed.
indices = [
gi for (gi, g) in enumerate(suffix_decomposition)
if g == grapheme
]
weight_dists = [
stats.norm(loc=stem_end + ind, scale=3.0)
for ind in indices
]
if len(weight_dists) == 0:
continue
for (hcol_n, hypercol) in enumerate(grapheme_hypercolumns):
#print('stem_end', stem_end, 'hcol', hcol_n, weight_dists[0].pdf(hcol_n))
nest.SetStatus(
nest.GetConnections(suffix_col,
hypercol[grapheme]),
{
'weight':
sum([
dist.pdf(hcol_n) for dist in weight_dists
]) * prm['suffix_grapheme_base_weight']
})
nest.Simulate(prm['letter_focus_time'])
class TnT(TaggerI):
'''
TnT - Statistical POS tagger
IMPORTANT NOTES:
* DOES NOT AUTOMATICALLY DEAL WITH UNSEEN WORDS
- It is possible to provide an untrained POS tagger to
create tags for unknown words, see __init__ function
* SHOULD BE USED WITH SENTENCE-DELIMITED INPUT
- Due to the nature of this tagger, it works best when
trained over sentence delimited input.
- However it still produces good results if the training
data and testing data are separated on all punctuation eg: [,.?!]
- Input for training is expected to be a list of sentences
where each sentence is a list of (word, tag) tuples
- Input for tag function is a single sentence
Input for tagdata function is a list of sentences
Output is of a similar form
* Function provided to process text that is unsegmented
- Please see basic_sent_chop()
TnT uses a second order Markov model to produce tags for
a sequence of input, specifically:
argmax [Proj(P(t_i|t_i-1,t_i-2)P(w_i|t_i))] P(t_T+1 | t_T)
IE: the maximum projection of a set of probabilities
The set of possible tags for a given word is derived
from the training data. It is the set of all tags
that exact word has been assigned.
To speed up and get more precision, we can use log addition
to instead multiplication, specifically:
argmax [Sigma(log(P(t_i|t_i-1,t_i-2))+log(P(w_i|t_i)))] +
log(P(t_T+1|t_T))
The probability of a tag for a given word is the linear
interpolation of 3 markov models; a zero-order, first-order,
and a second order model.
P(t_i| t_i-1, t_i-2) = l1*P(t_i) + l2*P(t_i| t_i-1) +
l3*P(t_i| t_i-1, t_i-2)
A beam search is used to limit the memory usage of the algorithm.
The degree of the beam can be changed using N in the initialization.
N represents the maximum number of possible solutions to maintain
while tagging.
It is possible to differentiate the tags which are assigned to
capitalized words. However this does not result in a significant
gain in the accuracy of the results.
'''
def __init__(self, unk=None, Trained=False, N=1000, C=False):
'''
Construct a TnT statistical tagger. Tagger must be trained
before being used to tag input.
:param unk: instance of a POS tagger, conforms to TaggerI
:type unk:(TaggerI)
:param Trained: Indication that the POS tagger is trained or not
:type Trained: boolean
:param N: Beam search degree (see above)
:type N:(int)
:param C: Capitalization flag
:type C: boolean
Initializer, creates frequency distributions to be used
for tagging
_lx values represent the portion of the tri/bi/uni taggers
to be used to calculate the probability
N value is the number of possible solutions to maintain
while tagging. A good value for this is 1000
C is a boolean value which specifies to use or
not use the Capitalization of the word as additional
information for tagging.
NOTE: using capitalization may not increase the accuracy
of the tagger
'''
self._uni = FreqDist()
self._bi = ConditionalFreqDist()
self._tri = ConditionalFreqDist()
self._wd = ConditionalFreqDist()
self._eos = ConditionalFreqDist()
self._l1 = 0.0
self._l2 = 0.0
self._l3 = 0.0
self._N = N
self._C = C
self._T = Trained
self._unk = unk
# statistical tools (ignore or delete me)
self.unknown = 0
self.known = 0
def train(self, data):
'''
Uses a set of tagged data to train the tagger.
If an unknown word tagger is specified,
it is trained on the same data.
:param data: List of lists of (word, tag) tuples
:type data: tuple(str)
'''
# Ensure that local C flag is initialized before use
C = False
if self._unk is not None and self._T == False:
self._unk.train(data)
for sent in data:
history = [('BOS',False), ('BOS',False)]
for w, t in sent:
# if capitalization is requested,
# and the word begins with a capital
# set local flag C to True
if self._C and w[0].isupper(): C=True
self._wd[w].inc(t)
self._uni.inc((t,C))
self._bi[history[1]].inc((t,C))
self._tri[tuple(history)].inc((t,C))
history.append((t,C))
history.pop(0)
# set local flag C to false for the next word
C = False
self._eos[t].inc('EOS')
# compute lambda values from the trained frequency distributions
self._compute_lambda()
#(debugging -- ignore or delete me)
#print "lambdas"
#print i, self._l1, i, self._l2, i, self._l3
def _compute_lambda(self):
'''
creates lambda values based upon training data
NOTE: no need to explicitly reference C,
it is contained within the tag variable :: tag == (tag,C)
for each tag trigram (t1, t2, t3)
depending on the maximum value of
- f(t1,t2,t3)-1 / f(t1,t2)-1
- f(t2,t3)-1 / f(t2)-1
- f(t3)-1 / N-1
increment l3,l2, or l1 by f(t1,t2,t3)
ISSUES -- Resolutions:
if 2 values are equal, increment both lambda values
by (f(t1,t2,t3) / 2)
'''
# temporary lambda variables
tl1 = 0.0
tl2 = 0.0
tl3 = 0.0
# for each t1,t2 in system
for history in self._tri.conditions():
(h1, h2) = history
# for each t3 given t1,t2 in system
# (NOTE: tag actually represents (tag,C))
# However no effect within this function
for tag in self._tri[history].samples():
# if there has only been 1 occurrence of this tag in the data
# then ignore this trigram.
if self._uni[tag] == 1:
continue
# safe_div provides a safe floating point division
# it returns -1 if the denominator is 0
c3 = self._safe_div((self._tri[history][tag]-1), (self._tri[history].N()-1))
c2 = self._safe_div((self._bi[h2][tag]-1), (self._bi[h2].N()-1))
c1 = self._safe_div((self._uni[tag]-1), (self._uni.N()-1))
# if c1 is the maximum value:
if (c1 > c3) and (c1 > c2):
tl1 += self._tri[history][tag]
# if c2 is the maximum value
elif (c2 > c3) and (c2 > c1):
tl2 += self._tri[history][tag]
# if c3 is the maximum value
elif (c3 > c2) and (c3 > c1):
tl3 += self._tri[history][tag]
# if c3, and c2 are equal and larger than c1
elif (c3 == c2) and (c3 > c1):
tl2 += float(self._tri[history][tag]) /2.0
tl3 += float(self._tri[history][tag]) /2.0
# if c1, and c2 are equal and larger than c3
# this might be a dumb thing to do....(not sure yet)
elif (c2 == c1) and (c1 > c3):
tl1 += float(self._tri[history][tag]) /2.0
tl2 += float(self._tri[history][tag]) /2.0
# otherwise there might be a problem
# eg: all values = 0
else:
#print "Problem", c1, c2 ,c3
pass
# Lambda normalisation:
# ensures that l1+l2+l3 = 1
self._l1 = tl1 / (tl1+tl2+tl3)
self._l2 = tl2 / (tl1+tl2+tl3)
self._l3 = tl3 / (tl1+tl2+tl3)
def _safe_div(self, v1, v2):
'''
Safe floating point division function, does not allow division by 0
returns -1 if the denominator is 0
'''
if v2 == 0:
return -1
else:
return float(v1) / float(v2)
def tagdata(self, data):
'''
Tags each sentence in a list of sentences
:param data:list of list of words
:type data: [[string,],]
:return: list of list of (word, tag) tuples
Invokes tag(sent) function for each sentence
compiles the results into a list of tagged sentences
each tagged sentence is a list of (word, tag) tuples
'''
res = []
for sent in data:
res1 = self.tag(sent)
res.append(res1)
return res
def tag(self, data):
'''
Tags a single sentence
:param data: list of words
:type data: [string,]
:return: [(word, tag),]
Calls recursive function '_tagword'
to produce a list of tags
Associates the sequence of returned tags
with the correct words in the input sequence
returns a list of (word, tag) tuples
'''
current_state = [(['BOS', 'BOS'], 0.0)]
sent = list(data)
tags = self._tagword(sent, current_state)
res = []
for i in range(len(sent)):
# unpack and discard the C flags
(t,C) = tags[i+2]
res.append((sent[i], t))
return res
def _tagword(self, sent, current_states):
'''
:param sent : List of words remaining in the sentence
:type sent : [word,]
:param current_states : List of possible tag combinations for
the sentence so far, and the log probability
associated with each tag combination
:type current_states : [([tag, ], logprob), ]
Tags the first word in the sentence and
recursively tags the reminder of sentence
Uses formula specified above to calculate the probability
of a particular tag
'''
# if this word marks the end of the sentance,
# return the most probable tag
if sent == []:
(h, logp) = current_states[0]
return h
# otherwise there are more words to be tagged
word = sent[0]
sent = sent[1:]
new_states = []
# if the Capitalisation is requested,
# initalise the flag for this word
C = False
if self._C and word[0].isupper(): C=True
# if word is known
# compute the set of possible tags
# and their associated log probabilities
if word in self._wd.conditions():
self.known += 1
for (history, curr_sent_logprob) in current_states:
logprobs = []
for t in self._wd[word].samples():
p_uni = self._uni.freq((t,C))
p_bi = self._bi[history[-1]].freq((t,C))
p_tri = self._tri[tuple(history[-2:])].freq((t,C))
p_wd = float(self._wd[word][t])/float(self._uni[(t,C)])
p = self._l1 *p_uni + self._l2 *p_bi + self._l3 *p_tri
p2 = log(p, 2) + log(p_wd, 2)
logprobs.append(((t,C), p2))
# compute the result of appending each tag to this history
for (tag, logprob) in logprobs:
new_states.append((history + [tag],
curr_sent_logprob + logprob))
# otherwise a new word, set of possible tags is unknown
else:
self.unknown += 1
# since a set of possible tags,
# and the probability of each specific tag
# can not be returned from most classifiers:
# specify that any unknown words are tagged with certainty
p = 1
# if no unknown word tagger has been specified
# then use the tag 'Unk'
if self._unk is None:
tag = ('Unk',C)
# otherwise apply the unknown word tagger
else :
[(_w, t)] = list(self._unk.tag([word]))
tag = (t,C)
for (history, logprob) in current_states:
history.append(tag)
new_states = current_states
# now have computed a set of possible new_states
# sort states by log prob
# set is now ordered greatest to least log probability
new_states.sort(reverse=True, key=itemgetter(1))
# del everything after N (threshold)
# this is the beam search cut
if len(new_states) > self._N:
new_states = new_states[:self._N]
# compute the tags for the rest of the sentence
# return the best list of tags for the sentence
return self._tagword(sent, new_states)
from nltk.draw.plot import Plot
freq_dist = FreqDist()
corpus = Token(TEXT=open('dados/may2001_pdf.torto').read())
print corpus
WhitespaceTokenizer().tokenize(corpus)
print corpus
for token in corpus['SUBTOKENS']:
freq_dist.inc(token['TEXT'])
# How many times did "the" occur?
freq_dist.count('the')
# What was the frequency of the word "the"?
freq_dist.freq('the')
# How many word tokens were counted?
freq_dist.N()
# What word types were encountered?
freq_dist.samples()
# What was the most common word?
freq_dist.max()
# What is the distribution of word lengths in a corpus?
freq_dist = FreqDist()
for token in corpus['SUBTOKENS']:
freq_dist.inc(len(token['TEXT']))
def get_entropy(category=None):
frq = FreqDist(brown.words(categories = category))
return sum(map(lambda w: -frq.freq(w)*math.log(frq.freq(w),2), frq.keys()))
# Qual é a distribuição do tamanho das palavras no corpus?
freq_dist = FreqDist()
for token in corpus['SUBTOKENS']:
freq_dist.inc(len(token['TEXT']))
# Desenha os resultados
wordlens = freq_dist.samples()
# Ordena a lista
wordlens.sort()
# cria uma tupla com um numero de frequencia e a sua
# respectiva distribuicao
# para visualizar execute o comanto print points
points = [(l, freq_dist.freq(l)) for l in wordlens]
Plot(points)
print points
# Qual é a distribuição do tamanho das palavras que terminal com
# vogais?
VOWELS = ('a', 'e', 'i', 'o', 'u')
freq_dist = FreqDist()
for token in corpus['SUBTOKENS']:
if token['TEXT'][-1].lower() in VOWELS:
freq_dist.inc(len(token['TEXT']))
# Desenha os resultados
wordlens = freq_dist.samples()
wordlens.sort()
dir(fdist)
fdist.max
fdist.values
fdist.values()
fdist.values().sum()
sum(fdist.values())
fdist['delicious'] / sum(fdist.values())
fdist['disgusting'] / sum(fdist.values())
fdist['disgusting']
fdist['vegetarian']
fdist['old-timey']
fdist['healthy']
fdist['expensive']
print text
print(text)
fdist.freq('delicious')
fdist.freq('delicnotehu')
fdist.N()
fdist ?
fdist?
fdist.freq('Delicious')
fdist
fdist.freq('rainy')
Business.where_raw('')
Business.where_raw('latitude <= 40.75')
Business.where_raw('latitude <= 40.75').count()
Business.where_raw('latitude <= 40.75 and latitude > 40.749')
Business.where_raw('latitude <= 40.75 and latitude > 40.749').count
Business.where_raw('latitude <= 40.75 and latitude > 40.749').count()
lat = 40.71
lon = -74.01
from nltk.token import *
from nltk.tokenizer import WhitespaceTokenizer
from nltk.probability import FreqDist
from nltk.draw.plot import Plot
freq_dist = FreqDist()
corpus = Token(TEXT=open('dados/may2001_pdf.torto').read())
print corpus
WhitespaceTokenizer().tokenize(corpus)
print corpus
for token in corpus['SUBTOKENS']:
freq_dist.inc(token['TEXT'])
# Quantas vezes a palavra form aparece no corpus?
freq_dist.count('form')
# Qual é a freqüência da palavra form?
freq_dist.freq('form')
# Quantas palavras foram contadas?
freq_dist.N()
# Quais foram os tipos de palavras encontradas?
freq_dist.samples()
# Qual é a palavra mais comum?
freq_dist.max()
#%%
from nltk.corpus import inaugural
from nltk import ConditionalFreqDist
from nltk.probability import FreqDist
fd3 = FreqDist([s for s in inaugural.words()])
print(fd3.freq('freedom'))
# count frequency of words length in decending order
cfd = ConditionalFreqDist((fileid, len(w)) for fileid in inaugural.fileids()
for w in inaugural.words(fileid)
if fileid > '1980' and fileid < '2010')
print(cfd.items())
cfd.plot()
# %%
def runTest(self,iteration):
print "running test %d"%iteration
pageUrl = '/reviews/www.zulily.com'
filename = '../test/resources/zulily.pkl'
try:
sjr = SiteJabberReviews(pageUrl,filename)
sjr.load()
helper = BayesHelper()
buckets = helper.generateLearningSetsFromReviews([sjr],[1,5],{'training': 0.8,'test':0.2})
self.assertEqual(len(buckets['training']), int(0.8*len(sjr.reviewsByRating[1])+int(0.8*len(sjr.reviewsByRating[5]))))
self.assertEqual(len(buckets['test']), int(0.2*len(sjr.reviewsByRating[1])+int(0.2*len(sjr.reviewsByRating[5]))))
# generate (term) tuples for FD -- this means we need to bust out like terms from combined distributions
allWords1 = [w for (textBag,rating) in buckets['training'] for w in textBag if rating == 1]
fd1 = FreqDist(allWords1)
allWords5 = [w for (textBag,rating) in buckets['training'] for w in textBag if rating == 5]
fd5 = FreqDist(allWords5)
commonTerms = [w for w in fd1.keys() if w in fd5.keys()]
commonTermFreqs = [(w,fd1.freq(w), fd5.freq(w), abs(fd1.freq(w) - fd5.freq(w))) for w in commonTerms]
commonTermFreqs.sort(key = itemgetter(3),reverse=True)
# commonDist = [freqDiff for (a,b,c,freqDiff) in commonTermFreqs]
#
# plt.plot(commonDist)
# plt.show()
# keep an arbitrary number
filterTerms = [w for (w,a,b,freq) in commonTermFreqs if freq > 0.001]
# add non common terms (note that bayesian will smooth zero terms out)
print 'high frequency differential featureset'
fd1Only = [w for w in fd1.keys() if w not in fd5.keys()]
filterTerms.extend(fd1Only)
fd5Only = [w for w in fd5.keys() if w not in fd1.keys()]
filterTerms.extend(fd5Only)
defaultWordSet = set(filterTerms)
def emitDefaultFeatures(tokenizedText):
'''
@param tokenizedText: an array of text features
@return: a feature map from that text.
'''
tokenizedTextSet = set(tokenizedText)
featureSet = {}
for text in defaultWordSet:
featureSet['contains:%s'%text] = text in tokenizedTextSet
return featureSet
classifier = None
encodedTrainSet = helper.encodeData(buckets['training'],emitDefaultFeatures )
classifier = nltk.NaiveBayesClassifier.train(encodedTrainSet)
encodedTestSet = helper.encodeData(buckets['test'], emitDefaultFeatures)
accuracy = nltk.classify.accuracy(classifier, encodedTestSet)
print "accuracy = %.9f"%accuracy
classifier.show_most_informative_features(10)
shouldBeClassed1 = []
shouldBeClassed5 = []
for (textbag, rating) in buckets['test']:
testRating = classifier.classify(emitDefaultFeatures(textbag))
if testRating != rating:
if rating == 1:
shouldBeClassed1.append(textbag)
else:
shouldBeClassed5.append(textbag)
print "length of mis-classified 1 star reviews = %d"%len(shouldBeClassed1)
print "length of mis-classified 5 star reviews = %d"%len(shouldBeClassed5)
print "length of all 1 star reviews submitted = %d"%len(sjr.reviewsByRating[1])
print "length of all 5 star reviews submitted = %d"%len(sjr.reviewsByRating[5])
print "length of test data for 1 star reviews = %d"%int(0.2*len(sjr.reviewsByRating[1]))
print "length of test data for 5 star reviews = %d"%int(0.2*len(sjr.reviewsByRating[5]))
# incorrectText1 = [(-1,w) for bag in shouldBeClassed1
# for w in bag if w not in stopwords.words('english')]
#
# correctText1 = [(1,w) for bag in buckets['training']
# for w in bag if w not in stopwords.words('english')]
#
# allText1 = []
# allText1.extend(incorrectText1)
# allText1.extend(correctText1)
#
# cfdText1 = ConditionalFreqDist(allText1)
#
#
# incorrectText5 = [(-5,w) for bag in shouldBeClassed5
# for w in bag if w not in stopwords.words('english')]
#
#
# correctText5 = [(5,w) for (bag, rating) in buckets['training']
# for w in bag if rating == 5 and w not in stopwords.words('english')]
#
#
# allText5 = []
# allText5.extend(incorrectText5)
# allText5.extend(correctText5)
#
# cfdText5 = ConditionalFreqDist(allText5)
return accuracy
except Exception as inst:
self.fail(inst)
