def fun14(): """counting other things""" # print [len(w) for w in text1] fdist1 = FreqDist([len(w) for w in text1]) # print fdist1.keys() # print fdist1.items() # word length 3 => 50223 print fdist1[3] print fdist1.max() # frequency 20% print fdist1.freq(3)
def binary_stump(feature_name, feature_value, labeled_featuresets): label = FreqDist([label for (featureset,label) in labeled_featuresets]).max() # Find the best label for each value. pos_fdist = FreqDist() neg_fdist = FreqDist() for featureset, label in labeled_featuresets: if featureset.get(feature_name) == feature_value: pos_fdist.inc(label) else: neg_fdist.inc(label) decisions = {feature_value: DecisionTreeClassifier(pos_fdist.max())} default = DecisionTreeClassifier(neg_fdist.max()) return DecisionTreeClassifier(label, feature_name, decisions, default)
def classify(self, feats): counts = FreqDist() for classifier in self._classifiers: counts.inc(classifier.classify(feats)) return counts.max()
def choose_tag(self, tokens, index, history): tags = FreqDist() for tagger in self._taggers: tags.inc(tagger.choose_tag(tokens, index, history)) return tags.max()
def choose_tag(self, tokens, index, history): word = tokens[index] fd = FreqDist() for synset in wordnet.synsets(word): fd.inc(synset.pos) return self.wordnet_tag_map.get(fd.max())
def classify(self, feat): '''Return the label with the most agreement among classifiers''' label_freqs = FreqDist() for classifier in self._classifiers: label_freqs.inc(classifier.classify(feat)) return label_freqs.max()
def shiftByAlpha(alphas, cipherText, common, reverse): key = [] for alpha in alphas: fdist = FreqDist(alpha) if reverse: shift = (ord(common) - ord(fdist.max())) else: shift = (ord(fdist.max()) - ord(common)) key.append(shift) print('shift ' + str(shift)) keyLen = len(key) res = '' for i in range(0, len(cipherText)): c = chr((ord(cipherText[i]) + key[i%keyLen])%128) res += c print (res)
def __compute_tf__(self, term, doc_terms): """ Computes the normalized frequency of term t in document d, which is the number of times t occurs in d divided by the maximum number of times any term occurs in d: tf(t,d) = f(t,d) / max{f(w,d)} """ fdist = FreqDist(term.lower() for term in doc_terms) max_freq = doc_terms.count(fdist.max()) if max_freq==0: return 0.0 return float(doc_terms.count(term)) / max_freq
def choose_tag(self, tokens, index, history): word = tokens[index] fd = FreqDist() for synset in wordnet.synsets(word): fd[synset.pos()] += 1 if not fd: return None return self.wordnet_tag_map.get(fd.max())
def binary_stump(feature_name, feature_value, labeled_featuresets): label = FreqDist(label for (featureset, label) in labeled_featuresets).max() # Find the best label for each value. pos_fdist = FreqDist() neg_fdist = FreqDist() for featureset, label in labeled_featuresets: if featureset.get(feature_name) == feature_value: pos_fdist[label] += 1 else: neg_fdist[label] += 1 decisions = {} default = label # But hopefully we have observations! if pos_fdist.N() > 0: decisions = {feature_value: DecisionTreeClassifier(pos_fdist.max())} if neg_fdist.N() > 0: default = DecisionTreeClassifier(neg_fdist.max()) return DecisionTreeClassifier(label, feature_name, decisions, default)
def choose_tag(self, tokens, index, history): word = tokens[index] if word is None: return None fd = FreqDist() for synset in wordnet.synsets(word): fd[synset.pos] += 1 try: return self.wordnet_tag_map.get(fd.max()) except: # in case fd is empty return None
def worst_errors_many_wrong_decisions(self, k, feature_extractor): worst_errors = [] features = [] wrongDocs = self.error_prediction_docs(self.maintest, self.testClassify) for doc in wrongDocs: feature_dic = feature_extractor(movie_reviews.words(fileids=[doc])) features = features + feature_dic.keys() fd = FreqDist(feature.lower() for feature in features) for i in range(1, k+1): x = fd.max() fd.pop(x) worst_errors.append(x) return worst_errors
def get_best_answers(self, passage_list, q): logger = logging.getLogger("qa_logger") logger.info("%s:\tAnswer Processing", q.id_q) empty = passage_list == [] logger.info("%s:\t\tAnswer Extraction", q.id_q) answer_list = [] for passage in passage_list: a = passage.find_answer(q) if a.is_successful(): answer_list.append(a) if not answer_list: return ([], empty) logger.info("%s:\t\tAnswer Filtering", q.id_q) # Obtain answer frequency fd = FreqDist(answer_list) # Normalize frequencies normalize = fd.freq(fd.max()) # Modify scores by frequency for answer in answer_list: answer.score = int(answer.score * (fd.freq(answer) / normalize)) # Sort answers by score answer_list.sort(key=lambda x: x.score, reverse=True) # Filter bad answers try: threshold = int(MyConfig.get("answer_filtering", "threshold")) except: logger = logging.getLogger("qa_logger") logger.error("answer quality threshold not found") threshold = 50 answer_list = filter(lambda x: x.score > threshold, answer_list) final_answers = [] for a in answer_list: if a not in final_answers: final_answers.append(a) if len(final_answers) == 3: break return (final_answers, empty)
def xorByAlpha(alphas, cipherText, common): key = [] for alpha in alphas: fdist = FreqDist(alpha) kxor = (ord(fdist.max()) ^ ord(common)) key.append(kxor) keyLen = len(key) res = '' for i in range(0, len(cipherText)): c = chr((ord(cipherText[i]) ^ key[i%keyLen])) res += c print (res)
def binary_stump(feature_name, feature_value, labeled_featuresets): label = FreqDist(label for (featureset, label) in labeled_featuresets).max() # Find the best label for each value. pos_fdist = FreqDist() neg_fdist = FreqDist() for featureset, label in labeled_featuresets: if featureset.get(feature_name) == feature_value: pos_fdist[label] += 1 else: neg_fdist[label] += 1 decisions = {} default = label # But hopefully we have observations! if pos_fdist.N() > 0: decisions = { feature_value: DecisionTreeClassifier(pos_fdist.max()) } if neg_fdist.N() > 0: default = DecisionTreeClassifier(neg_fdist.max()) return DecisionTreeClassifier(label, feature_name, decisions, default)
def _entity_ranking(self, entities): if len(entities) == 0: return "", "", int(0) # Obtain frequency of entities entities_freq = FreqDist(entities) # Our answer is the sample with the greatest number of outcomes exact = entities_freq.max() # Our window is empty because this algorithm generates exact answers window = "" # Our score is the entity frequency score = int(entities_freq.freq(exact) * 1000) return exact, window, score
def _entity_ranking(self, entities): if len(entities) == 0: return "", "", int(0) # Obtain frequency of entities entities_freq = FreqDist(entities) # Our answer is the sample with the greatest number of outcomes exact = entities_freq.max() # Our window is empty because this algorithm generates exact answers window = "" # Our score is the entity frequency score = int(entities_freq.freq(exact) * 1000) return exact, window, score
def choose_tag(self, tokens, index, history): context = self.context(tokens, index, history) s = self._morph.parse(tokens[index]) tags = [unicode(x.tag).replace(u' ', u',') for x in s] if len(tags) == 0: return None if (len(tags) == 1) or not (context in self._contexts_to_tags.keys()): return tags[0] tagsconts = FreqDist() for tag in tags: #print 'TAG: ', tag #print tokens[index] tagsconts[tag] = self._contexts_to_tags[context].get(tag, 0) #print 'PROB: | ', context, tagsconts[tag] best_tag = tagsconts.max() if tagsconts[best_tag] == 0: return tags[0] return best_tag
def next(self, s, method = MOST_LIKELY): # Pick a transition leaving state s and return a state that would # likely follow. The next state is chosen according to the method # specified. The default is to choose and return the most likely # transition state. # determine all states adjacent to s transitions = self._adjacentVertices[s] freqDist = FreqDist() # determine the weights of the edges between state s and all adjacent states for state in transitions: freqDist.inc(state) if method == MarkovChain.MOST_LIKELY: return freqDist.max() elif method == MarkovChain.LEAST_LIKELY: # NLTK provides no built-in method to return the minimum of a # frequency distribution so for now, we get a list of samples # sorted in decreasing order and grab the last one. return freqDist.sorted_samples()[-1] else: # choose a real number between 0 and 1 x = uniform(0,1) # choose next state based on weights of the edges. Randomness plays a part here. for i in range(len(transitions)): probability = freqDist.freq(transitions[i]) if x < probability: return transitions[i] x = x - probability exc = "Error in MarkovChain.next(). Did not find next state.\n" raise exc
class WordNetTagger(SequentialBackoffTagger): """ Class implementation of the wordnet tagger """ def __init__(self, *args, **kwargs): SequentialBackoffTagger.__init__(self, *args, **kwargs) self.wordnet_tag_map = { 'n': 'NN', 's': 'JJ', 'a': 'JJ', 'r': 'RB', 'v': 'VB' } self.fd = FreqDist(treebank.words()) def choose_tag(self, tokens, index, history): """ Choses a POS tag based on the wordnet tag """ word = tokens[index] for synset in wordnet.synsets(word): self.fd[synset.pos()] += 1 return self.wordnet_tag_map.get(self.fd.max())
print("2.", len(cess_esp.words())) # 3 print("3.", len(cess_esp.sents())) # 4 from nltk.probability import FreqDist first_file = cess_esp.fileids()[0] cess_freq0 = FreqDist(cess_esp.words(first_file)) print("4.", cess_freq0.most_common(20)) # 5 print("5.", [w for w, k in cess_freq0.most_common()]) # 6 print("6.", [w for w, k in cess_freq0.items() if len(w) > 7 and k > 2]) # 7 print("7.", [k for w, k in cess_freq0.most_common()]) print("7b. Freq de aparición de la preposición a", cess_freq0.get("a", 0)) # 8 print("8. No de palabras que aparecen una sola vez:", len([w for w, k in cess_freq0.items() if k == 1])) # 9 print("9. La palabra más frecuente es", cess_freq0.max()) # 10 from nltk.corpus import PlaintextCorpusReader mycorpus = PlaintextCorpusReader("../res/", ".*") # 11 print("11.") for doc in mycorpus.fileids(): print(doc, len(mycorpus.words(doc)), len(set(mycorpus.words(doc))), len(mycorpus.sents(doc)))
freq_dist.inc(token['TEXT']) # How many times did "the" occur? freq_dist.count('the') # What was the frequency of the word "the"? freq_dist.freq('the') # How many word tokens were counted? freq_dist.N() # What word types were encountered? freq_dist.samples() # What was the most common word? freq_dist.max() # What is the distribution of word lengths in a corpus? freq_dist = FreqDist() for token in corpus['SUBTOKENS']: freq_dist.inc(len(token['TEXT'])) # Plot the results. wordlens = freq_dist.samples() # Ordena a lista wordlens.sort() # cria uma tupla com um numero de frequencia e a sua # respectiva distribuicao # para visualizar execute o comanto print points
def default_tag(tagged_sents): tag_fd = FreqDist() for sent in tagged_sents: for word, postag in sent: tag_fd.inc(postag) return str(tag_fd.max())
from nltk.corpus import cess_esp from nltk.probability import FreqDist fdist = FreqDist(cess_esp.words(cess_esp.fileids()[0])) print("La palabra mas frecuente es ", fdist.max())
def classify(self, feats): counts = FreqDist() for classifier in self._classifiers: counts[classifier.classify(feats)] += 1 return counts.max()
#print "lines: ", len(lines) for line in lines: # print n, line.encode('utf-8') line_tokens = tokenizer.tokenize(line) #for token in line_tokens: # print token.encode('utf-8'), " | " #n = n + 1 text_array.append(line_tokens) #now try to match hyphenated lines with their #correpsonding beginning lines n = 0 for line in text_array: if len(line) > 0: if line[-1][-1] == '-': try: line[-1] = line[-1][:-1] + text_array[n + 1][0] text_array[n + 1] = text_array[n + 1][1:] except IndexError as e: print e n = n + 1 #now flatten the 2d array tokens = [item for sublist in text_array for item in sublist] tokens = delete_non_greek_tokens(tokens) for token in tokens: fdist.inc(token) print "most common: ", fdist.max().encode('utf-8') for item in fdist.keys(): print item.encode('utf-8'), fdist.freq(item)
wc += 1 # loading corpora/treebank/tagged with ChunkedCorpusReader produces None tags if not isinstance(tag, basestring): tag = str(tag) tag_counts.inc(tag) word_set.add(word) ############ ## output ## ############ print '%d total words\n%d unique words\n%d tags\n' % (wc, len(word_set), len(tag_counts)) if args.sort == 'tag': sort_key = lambda (t, c): t elif args.sort == 'count': sort_key = lambda (t, c): c else: raise ValueError('%s is not a valid sort option' % args.sort) countlen = max(len(str(tag_counts[tag_counts.max()])) + 2, 9) # simple reSt table format print ' '.join(['Tag'.center(taglen), 'Count'.center(countlen)]) print ' '.join(['=' * taglen, '=' * (countlen)]) for tag, count in sorted(tag_counts.items(), key=sort_key, reverse=args.reverse): print ' '.join([tag.ljust(taglen), str(count).rjust(countlen)]) print ' '.join(['=' * taglen, '=' * (countlen)])
cfd = ConditionalFreqDist() ### get the (token,tag) pair for each tagged sentence i = 1 for sentence in brown.tagged_sents(): for (token, tag) in sentence: if i < 6: print(token, tag) fd.inc(tag) cfd[token].inc(tag) i += 1 ### the most frequent tag: print fd.max() wordbins = [] for token in cfd.conditions(): wordbins.append((cfd[token].B(), token)) ### sort tuples by number of unique tags wordbins.sort(reverse=True) print wordbins[0:3] ### masculine pronouns male = ['he', 'his', 'him', 'himself'] female = ['she', 'hers', 'her', 'herself'] n_male, n_female = 0, 0
if args.corpus in ['conll2000', 'switchboard'] and simplify_wsj_tag and args.simplify_tags: tag = simplify_wsj_tag(tag) wc += 1 # loading corpora/treebank/tagged with ChunkedCorpusReader produces None tags if not isinstance(tag, basestring): tag = str(tag) tag_counts.inc(tag) word_set.add(word) ############ ## output ## ############ print('%d total words\n%d unique words\n%d tags\n' % (wc, len(word_set), len(tag_counts))) if args.sort == 'tag': sort_key = lambda tc: tc[0] elif args.sort == 'count': sort_key = lambda tc: tc[1] else: raise ValueError('%s is not a valid sort option' % args.sort) countlen = max(len(str(tag_counts[tag_counts.max()])) + 2, 9) # simple reSt table format print(' '.join(['Tag'.center(taglen), 'Count'.center(countlen)])) print(' '.join(['='*taglen, '='*(countlen)])) for tag, count in sorted(tag_counts.items(), key=sort_key, reverse=args.reverse): print(' '.join([tag.ljust(taglen), str(count).rjust(countlen)])) print(' '.join(['='*taglen, '='*(countlen)]))
def choose_tag(self, tokens, index, history): word = tokens[index] fd = FreqDist() for synset in wordnet.synsets(word): fd.inc(synset.pos) return self.wordnet_tag_map.get(fd.max())
def default_tag(tagged_sents): tag_fd = FreqDist() for sent in tagged_sents: for word, postag in sent: tag_fd.inc(postag) return str(tag_fd.max())
# print "lines: ", len(lines) for line in lines: # print n, line.encode('utf-8') line_tokens = tokenizer.tokenize(line) # for token in line_tokens: # print token.encode('utf-8'), " | " # n = n + 1 text_array.append(line_tokens) # now try to match hyphenated lines with their # correpsonding beginning lines n = 0 for line in text_array: if len(line) > 0: if line[-1][-1] == "-": try: line[-1] = line[-1][:-1] + text_array[n + 1][0] text_array[n + 1] = text_array[n + 1][1:] except IndexError as e: print e n = n + 1 # now flatten the 2d array tokens = [item for sublist in text_array for item in sublist] tokens = delete_non_greek_tokens(tokens) for token in tokens: fdist.inc(token) print "most common: ", fdist.max().encode("utf-8") for item in fdist.keys(): print item.encode("utf-8"), fdist.freq(item)