def test_sentiment(self): # Assert < 0 for negative adjectives and > 0 for positive adjectives. self.assertTrue(en.sentiment("wonderful")[0] > 0) self.assertTrue(en.sentiment("horrible")[0] < 0) self.assertTrue(en.sentiment(en.wordnet.synsets("horrible", pos="JJ")[0])[0] < 0) self.assertTrue(en.sentiment(en.Text(en.parse("A bad book. Really horrible.")))[0] < 0) # Assert that :) and :( are recognized. self.assertTrue(en.sentiment(":)")[0] > 0) self.assertTrue(en.sentiment(":(")[0] < 0) # Assert the accuracy of the sentiment analysis (for the positive class). # Given are the scores for Pang & Lee's polarity dataset v2.0: # http://www.cs.cornell.edu/people/pabo/movie-review-data/ # The baseline should increase (not decrease) when the algorithm is modified. from pattern.db import Datasheet from pattern.metrics import test reviews = [] for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee1.csv")): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: en.positive(review), reviews) self.assertTrue(A > 0.755) self.assertTrue(P > 0.760) self.assertTrue(R > 0.747) self.assertTrue(F > 0.754) # Assert the accuracy of the sentiment analysis on short text (for the positive class). # Given are the scores for Pang & Lee's sentence polarity dataset v1.0: # http://www.cs.cornell.edu/people/pabo/movie-review-data/ reviews = [] for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee2.csv")): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: en.positive(review), reviews) self.assertTrue(A > 0.642) self.assertTrue(P > 0.653) self.assertTrue(R > 0.607) self.assertTrue(F > 0.629) print "pattern.en.sentiment()"
def testModel(self, *args): ''' Perform learning of a Model from training data. ''' documents = [] data = Datasheet.load(os.path.join("corpora","twitter","trainer", "tweets_stream_data_nb.csv")) data2 = Datasheet.load(os.path.join("corpora","twitter","trainer", "tweets_stream_data_svm.csv")) data3 = Datasheet.load(os.path.join("corpora","twitter","trainer", "ensenmble.csv")) if args: classifier = Classifier.load('models/nb_model.ept') print "Document class is %s" % classifier.classify(Document(args[0])) print "Document probability is : ", classifier.classify(Document(args[0]), discrete=False) label = classifier.classify(Document(args[0]), discrete=False) print label["positive"] else: i = n = 0 pos=neg=0 classifier = Classifier.load('models/nb_model.ept') data = shuffled(data) for document, label in data[:]+data2[:]+data3[:]: doc_vector = Document(document, type=str(label), stopwords=True) documents.append(doc_vector) if 'positive' in label: pos+=1 else: neg+=1 print "10-fold CV" print k_fold_cv(NB, documents=documents, folds=10) print "Neg: %s, Pos: %s" % (neg, pos) print classifier.distribution print "Classes in Naive Bayes Classifier" print classifier.classes print "Area Under the Curve: %0.6f" % classifier.auc(documents, k=10) print "Model Performance (Positive Classifications)" accuracy, precision, recall, f1 = classifier.test(data[:]+data2[:]+data3[:], target='positive') print "Accuracy = %.6f; F-Score = %.6f; Precision = %.6f; Recall = %.6f" % (accuracy, f1, precision, recall) print "Model Performance(Negative Classifications)" accuracy, precision, recall, f1 = classifier.test(data[:]+data2[:]+data3[:], target='negative') print "Accuracy = %.6f; F-Score = %.6f; Precision = %.6f; Recall = %.6f" % (accuracy, f1, precision, recall) print "Model Performance" accuracy, precision, recall, f1 = classifier.test(data[:]+data2[:]+data3[:]) print "Accuracy = %.6f; F-Score = %.6f; Precision = %.6f; Recall = %.6f" % (accuracy, f1, precision, recall) print "Confusion Matrix" print classifier.confusion_matrix(data[:]+data2[:]+data3[:]) print classifier.confusion_matrix(data[:]+data2[:]+data3[:])('positive') print classifier.confusion_matrix(data[:]+data2[:]+data3[:])('negative')
def train(self, train_path): """ Train classifier on features from headline and article text """ if self.debug: tick = time() logging.info("Training new model with %s" % (train_path,)) logging.info("Loading/shuffling training data...") train_data_1 = Datasheet.load(train_path) shuffle(train_data_1) train_texts_1 = zip(train_data_1.columns[0], train_data_1.columns[1]) train_labels_1 = [0 if x == '0' else 1 for x in train_data_1.columns[-1]] if self.debug: logging.info('Fitting training data') pipeline_1 = self.create_pipeline() pipeline_1.fit(train_texts_1, train_labels_1) if self.debug: logging.info("Done in %0.2fs" % (time() - tick,)) train_data_2 = Datasheet() for row in train_data_1.rows: if row[-1] != '0': train_data_2.append(row) train_texts_2 = zip(train_data_2.columns[0], train_data_2.columns[1]) train_labels_2 = train_data_2.columns[-1] pipeline_2 = self.create_pipeline() pipeline_2.fit(train_texts_2, train_labels_2) return pipeline_1, pipeline_2
def test_sentiment_twitter(self): sanders = os.path.join(PATH, "corpora", "polarity-en-sanders.csv") if os.path.exists(sanders): # Assert the accuracy of the sentiment analysis on tweets. # Given are the scores for Sanders Twitter Sentiment Corpus: # http://www.sananalytics.com/lab/twitter-sentiment/ # Positive + neutral is taken as polarity >= 0.0, # Negative is taken as polarity < 0.0. # Since there are a lot of neutral cases, # and the algorithm predicts 0.0 by default (i.e., majority class) the results are good. # Distinguishing negative from neutral from positive is a much # harder task from pattern.db import Datasheet from pattern.metrics import test reviews = [] for i, id, date, tweet, polarity, topic in Datasheet.load(sanders): if polarity != "irrelevant": reviews.append( (tweet, polarity in ("positive", "neutral"))) A, P, R, F = test( lambda review: en.positive(review, threshold=0.0), reviews) #print(A, P, R, F) self.assertTrue(A > 0.824) self.assertTrue(P > 0.879) self.assertTrue(R > 0.911) self.assertTrue(F > 0.895)
def test_sentiment(self): # Assert < 0 for negative adjectives and > 0 for positive adjectives. self.assertTrue(en.sentiment("wonderful")[0] > 0) self.assertTrue(en.sentiment("horrible")[0] < 0) self.assertTrue( en.sentiment(en.wordnet.synsets("horrible", pos="JJ")[0])[0] < 0) self.assertTrue( en.sentiment(en.Text(en.parse("A bad book. Really horrible.")))[0] < 0) # Assert the accuracy of the sentiment analysis. # Given are the scores for Pang & Lee's polarity dataset v2.0: # http://www.cs.cornell.edu/people/pabo/movie-review-data/ # The baseline should increase (not decrease) when the algorithm is modified. from pattern.db import Datasheet from pattern.metrics import test reviews = [] for score, review in Datasheet.load( os.path.join("corpora", "pang&lee-polarity.txt")): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: en.positive(review), reviews) self.assertTrue(A > 0.71) self.assertTrue(P > 0.72) self.assertTrue(R > 0.70) self.assertTrue(F > 0.71) print "pattern.en.sentiment()"
def test_spelling(self): # Assert case-sensitivity + numbers. for a, b in ( (".", "."), ("?", "?"), ("!", "!"), ("I", "I"), ("a", "a"), ("42", "42"), ("3.14", "3.14"), ("The", "The"), ("the", "the")): self.assertEqual(en.suggest(a)[0][0], b) # Assert spelling suggestion accuracy. # Note: simply training on more text will not improve accuracy. i = j = 0.0 from pattern.db import Datasheet for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv")): for w in wrong.split(" "): if en.suggest(w)[0][0] == correct: i += 1 else: j += 1 self.assertTrue(i / (i + j) > 0.70) print("pattern.en.suggest()")
def test_modality(self): # Assert -1.0 => +1.0 representing the degree of certainty. v = en.modality(en.Sentence(en.parse("I wish it would stop raining."))) self.assertTrue(v < 0) v = en.modality( en.Sentence(en.parse("It will surely stop raining soon."))) self.assertTrue(v > 0) # Assert the accuracy of the modality algorithm. # Given are the scores for the CoNLL-2010 Shared Task 1 Wikipedia uncertainty data: # http://www.inf.u-szeged.hu/rgai/conll2010st/tasks.html#task1 # The baseline should increase (not decrease) when the algorithm is modified. from pattern.db import Datasheet from pattern.metrics import test sentences = [] for certain, sentence in Datasheet.load( os.path.join(PATH, "corpora", "uncertainty-conll2010.csv")): sentence = en.parse(sentence, chunks=False, light=True) sentence = en.Sentence(sentence) sentences.append((sentence, int(certain) > 0)) A, P, R, F = test(lambda sentence: en.modality(sentence) > 0.5, sentences) #print A, P, R, F self.assertTrue(A > 0.69) self.assertTrue(P > 0.71) self.assertTrue(R > 0.64) self.assertTrue(F > 0.67) print "pattern.en.modality()"
def test_sentiment_twitter(self): sanders = os.path.join(PATH, "corpora", "polarity-en-sanders.csv") if os.path.exists(sanders): # Assert the accuracy of the sentiment analysis on tweets. # Given are the scores for Sanders Twitter Sentiment Corpus: # http://www.sananalytics.com/lab/twitter-sentiment/ # Positive + neutral is taken as polarity >= 0.0, # Negative is taken as polarity < 0.0. # Since there are a lot of neutral cases, # and the algorithm predicts 0.0 by default (i.e., majority class) the results are good. # Distinguishing negative from neutral from positive is a much harder task from pattern.db import Datasheet from pattern.metrics import test reviews = [] for i, id, date, tweet, polarity, topic in Datasheet.load(sanders): if polarity != "irrelevant": reviews.append((tweet, polarity in ("positive", "neutral"))) A, P, R, F = test( lambda review: en.positive(review, threshold=0.0), reviews) #print A, P, R, F self.assertTrue(A > 0.824) self.assertTrue(P > 0.879) self.assertTrue(R > 0.911) self.assertTrue(F > 0.895)
def test_spelling(self): # Assert case-sensitivity + numbers. for a, b in ( ( ".", "." ), ( "?", "?" ), ( "!", "!" ), ( "I", "I" ), ( "a", "a" ), ( "42", "42" ), ("3.14", "3.14"), ( "The", "The" ), ( "the", "the" )): self.assertEqual(en.suggest(a)[0][0], b) # Assert spelling suggestion accuracy. # Note: simply training on more text will not improve accuracy. i = j = 0.0 from pattern.db import Datasheet for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv")): for w in wrong.split(" "): if en.suggest(w)[0][0] == correct: i += 1 else: j += 1 self.assertTrue(i / (i + j) > 0.70) print("pattern.en.suggest()")
def load_domains(self): sources_path = pd('data', 'source_data.csv') domain_file = Datasheet.load(sources_path, headers=True) for row in domain_file: url = row[1] cats = row[2:] self.cat_dict[url] = cats
def test_modality(self): # Assert -1.0 => +1.0 representing the degree of certainty. v = en.modality(en.Sentence(en.parse("I wish it would stop raining."))) self.assertTrue(v < 0) v = en.modality( en.Sentence(en.parse("It will surely stop raining soon."))) self.assertTrue(v > 0) # Assert the accuracy of the modality algorithm. # Given are the scores for the CoNLL-2010 Shared Task 1 Wikipedia uncertainty data: # http://www.inf.u-szeged.hu/rgai/conll2010st/tasks.html#task1 # The baseline should increase (not decrease) when the algorithm is # modified. from pattern.db import Datasheet from pattern.metrics import test sentences = [] for certain, sentence in Datasheet.load(os.path.join(PATH, "corpora", "uncertainty-conll2010.csv")): sentence = en.parse(sentence, chunks=False, light=True) sentence = en.Sentence(sentence) sentences.append((sentence, int(certain) > 0)) A, P, R, F = test( lambda sentence: en.modality(sentence) > 0.5, sentences) #print(A, P, R, F) self.assertTrue(A > 0.69) self.assertTrue(P > 0.72) self.assertTrue(R > 0.64) self.assertTrue(F > 0.68) print("pattern.en.modality()")
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")): if en.inflect.singularize(pl) == sg: i +=1 n += 1 self.assertTrue(float(i) / n > 0.95) print "pattern.en.inflect.singularize()"
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): if it.singularize(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.84) print "pattern.it.singularize()"
def model(top=None): """ Returns a Model of e-mail messages. Document type=True => HAM, False => SPAM. Documents are mostly of a technical nature (developer forum posts). """ documents = [] for score, message in Datasheet.load(os.path.join(PATH, "corpora", "spam-apache.csv")): document = vector.Document(message, stemmer="porter", top=top, type=int(score) > 0) documents.append(document) return vector.Model(documents)
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): if it.singularize(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.84) print("pattern.it.singularize()")
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): if nl.singularize(pl) == sg: i +=1 n += 1 self.assertTrue(float(i) / n > 0.88) print "pattern.nl.singularize()"
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("felle" => "fel"). from pattern.db import Datasheet i, n = 0, 0 for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): if nl.predicative(attr) == pred: i +=1 n += 1 self.assertTrue(float(i) / n > 0.96) print "pattern.nl.predicative()"
def test_attributive(self): # Assert the accuracy of the attributive algorithm ("fel" => "felle"). from pattern.db import Datasheet i, n = 0, 0 for pred, attr, sg, pl in Datasheet.load(os.path.join("corpora", "celex-wordforms-nl.csv")): if nl.attributive(pred) == attr: i +=1 n += 1 self.assertTrue(float(i) / n > 0.96) print "pattern.nl.attributive()"
def test_pluralize(self): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): if it.pluralize(sg) == pl: i += 1 n += 1 self.assertTrue(float(i) / n > 0.93) print("pattern.it.pluralize()")
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")): if en.inflect.singularize(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.95) print("pattern.en.inflect.singularize()")
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): if nl.singularize(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.88) print("pattern.nl.singularize()")
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("felle" => "fel"). from pattern.db import Datasheet i, n = 0, 0 for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): if nl.predicative(attr) == pred: i += 1 n += 1 self.assertTrue(float(i) / n > 0.96) print("pattern.nl.predicative()")
def main(): logging.basicConfig(level=logging.INFO) argparser = ArgumentParser(description=__doc__) argparser.add_argument("-t", "--trainset", action="store", default=None, help=("Path to training data " "[default: %(default)s]")) argparser.add_argument("-m", "--model", action="store", help="Path to model") argparser.add_argument("-d", "--dump", action="store_true", help="Pickle trained model? [default: False]") argparser.add_argument("-v", "--verbose", action="store_true", default=False, help="Verbose [default: quiet]") argparser.add_argument("-c", "--classify", action="store", default=None, help=("Path to data to classify " "[default: %(default)s]")) argparser.add_argument("-s", "--save", action="store", default='output.csv', help=("Path to output file" "[default = output.csv]")) args = argparser.parse_args() clf = SensationalismClassifier(train_data=args.trainset, model=args.model, dump=args.dump, debug=args.verbose) if args.classify: OUTPUT_PATH = args.save if clf.debug: tick = time() to_classify = Datasheet.load(args.classify) classified_data = clf.classify(to_classify) output = Datasheet(classified_data) output.save(pd(OUTPUT_PATH)) if clf.debug: sys.stderr.write("\nProcessed %d items in %0.2fs" % (len(classified_data), time() - tick))
def scrape_news_text(news_url): global counter news_html = requests.get(news_url).content # print(news_html) '''convert html to BeautifulSoup object''' news_soup = BeautifulSoup(news_html, 'lxml') # soup.find("div", {"id": "articlebody"}) # paragraphs = [par.text for par in news_soup.find_all('p')] # news_text = '\n'.join(paragraphs) # print(news_soup.find("div", {"id": "articleText"})) date_object = news_soup.find(itemprop="datePublished") news_object = news_soup.find("div", {"id": "articleText"}) if date_object is None: return " " if news_object is None: return " " news_date = date_object.get_text( ) # find("div", {"id": "articleText"}).text news_text = news_object.text # print(news_date) # print(news_text) print(news_url) try: # We'll store tweets in a Datasheet. # A Datasheet is a table of rows and columns that can be exported as a CSV-file. # In the first column, we'll store a unique id for each tweet. # We only want to add the latest tweets, i.e., those we haven't seen yet. # With an index on the first column we can quickly check if an id already exists. # The pd() function returns the parent directory of this script + any given path. table = Datasheet.load(pd("nasdaq2.csv")) except: table = Datasheet() news_sentiment = sentiment(news_text) print(news_sentiment) table.append([counter, news_date, news_url, news_sentiment]) table.save(pd("nasdaq2.csv")) counter += 1 return news_text
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 for tag, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "celex-wordforms-de.csv")): if tag == "n": if de.singularize(pl) == sg: i +=1 n += 1 self.assertTrue(float(i) / n > 0.81) print "pattern.de.singularize()"
def test_pluralize(self): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for tag, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-de-celex.csv")): if tag == "n": if de.pluralize(sg) == pl: i +=1 n += 1 self.assertTrue(float(i) / n > 0.69) print "pattern.de.pluralize()"
def test_intertextuality(self): # Evaluate accuracy for plagiarism detection. from pattern.db import Datasheet data = Datasheet.load(os.path.join(PATH, "corpora", "plagiarism-clough&stevenson.csv")) data = [((txt, src), int(plagiarism) > 0) for txt, src, plagiarism in data] def plagiarism(txt, src): return metrics.intertextuality([txt, src], n=3)[0,1] > 0.05 A, P, R, F = metrics.test(lambda x: plagiarism(*x), data) self.assertTrue(P > 0.96) self.assertTrue(R > 0.94) print "pattern.metrics.intertextuality()"
def test_spelling(self): # Assert spelling suggestion accuracy. i = j = 0.0 from pattern.db import Datasheet for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "birkbeck-spelling.csv")): for w in wrong.split(" "): if en.spelling(w)[0][0] == correct: i += 1 else: j += 1 self.assertTrue(i / (i+j) > 0.70)
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("belles" => "beau"). from pattern.db import Datasheet i, n = 0, 0 for pred, attr, tag in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-fr-lexique.csv")): if tag == "a": if fr.predicative(attr) == pred: i +=1 n += 1 self.assertTrue(float(i) / n > 0.95) print "pattern.fr.predicative()"
def test_intertextuality(self): # Evaluate accuracy for plagiarism detection. from pattern.db import Datasheet data = Datasheet.load(os.path.join(PATH, "corpora", "plagiarism-clough&stevenson.csv")) data = [((txt, src), int(plagiarism) > 0) for txt, src, plagiarism in data] def plagiarism(txt, src): return metrics.intertextuality([txt, src], n=3)[0,1] > 0.05 A, P, R, F = metrics.test(lambda x: plagiarism(*x), data) self.assertTrue(P > 0.96) self.assertTrue(R > 0.94) print("pattern.metrics.intertextuality()")
def test_pluralize(self): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load( os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): if it.pluralize(sg) == pl: i += 1 n += 1 self.assertTrue(float(i) / n > 0.93) print "pattern.it.pluralize()"
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("großer" => "groß"). from pattern.db import Datasheet i, n = 0, 0 for tag, pred, attr in Datasheet.load(os.path.join(PATH, "corpora", "celex-wordforms-de.csv")): if tag == "a": if de.predicative(attr) == pred: i +=1 n += 1 self.assertTrue(float(i) / n > 0.98) print "pattern.de.predicative()"
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("großer" => "groß"). from pattern.db import Datasheet i, n = 0, 0 for tag, pred, attr in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-de-celex.csv")): if tag == "a": if de.predicative(attr) == pred: i +=1 n += 1 self.assertTrue(float(i) / n > 0.98) print("pattern.de.predicative()")
def test_pluralize(self): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for tag, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-de-celex.csv")): if tag == "n": if de.pluralize(sg) == pl: i +=1 n += 1 self.assertTrue(float(i) / n > 0.69) print("pattern.de.pluralize()")
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("belles" => "beau"). from pattern.db import Datasheet i, n = 0, 0 for pred, attr, tag in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-fr-lexique.csv")): if tag == "a": if fr.predicative(attr) == pred: i +=1 n += 1 self.assertTrue(float(i) / n > 0.95) print("pattern.fr.predicative()")
def load_domains(self): """loads domain information""" sources_path = pd('data', 'source_data.csv') domain_file = Datasheet.load(sources_path, headers=True) for row in domain_file: url = row[1] if str(row[-1]).find("\""): cats = row[2:-1] else: cats = row[2:] self.cat_dict[url] = cats
def test_pluralize(self): # Assert "auto's" as plural of "auto". self.assertEqual("auto's", nl.inflect.pluralize("auto")) # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): if nl.pluralize(sg) == pl: i += 1 n += 1 self.assertTrue(float(i) / n > 0.74) print("pattern.nl.pluralize()")
def test_pluralize(self): # Assert "auto's" as plural of "auto". self.assertEqual("auto's", nl.inflect.pluralize("auto")) # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): if nl.pluralize(sg) == pl: i +=1 n += 1 self.assertTrue(float(i) / n > 0.74) print "pattern.nl.pluralize()"
def test_spelling(self): # Assert spelling suggestion accuracy. # Note: simply training on more text will not improve accuracy. i = j = 0.0 from pattern.db import Datasheet for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "birkbeck-spelling.csv")): for w in wrong.split(" "): if en.spelling(w)[0][0] == correct: i += 1 else: j += 1 self.assertTrue(i / (i+j) > 0.70)
def test_spelling(self): # Assert spelling suggestion accuracy. # Note: simply training on more text will not improve accuracy. i = j = 0.0 from pattern.db import Datasheet for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv")): for w in wrong.split(" "): if en.spelling(w)[0][0] == correct: i += 1 else: j += 1 self.assertTrue(i / (i+j) > 0.70)
def test_spelling(self): i = j = 0.0 from pattern.db import Datasheet for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-ru.csv")): for w in wrong.split(" "): suggested = ru.suggest(w) if suggested[0][0] == correct: i += 1 else: j += 1 self.assertTrue(i / (i + j) > 0.65) print("pattern.ru.suggest()")
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("cruciali" => "cruciale"). from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): if pos != "j": continue if it.predicative(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.87) print "pattern.it.predicative()"
def test_gender(self): # Assert the accuracy of the gender disambiguation algorithm. from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): g = it.gender(sg) if mf in g and it.PLURAL not in g: i += 1 g = it.gender(pl) if mf in g and it.PLURAL in g: i += 1 n += 2 self.assertTrue(float(i) / n > 0.92) print "pattern.it.gender()"
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("cruciali" => "cruciale"). from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load( os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): if pos != "j": continue if it.predicative(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.87) print("pattern.it.predicative()")
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet test = {} for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")): if tag == "n": test.setdefault(lemma, []).append(w) i, n = 0, 0 for sg, pl in test.items(): pl = sorted(pl, key=len, reverse=True)[0] if es.singularize(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.93) print "pattern.es.singularize()"
def test_pluralize(self): # Assert "octopodes" for classical plural of "octopus". # Assert "octopuses" for modern plural. self.assertEqual("octopodes", en.inflect.pluralize("octopus", classical=True)) self.assertEqual("octopuses", en.inflect.pluralize("octopus", classical=False)) # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")): if en.inflect.pluralize(sg) == pl: i += 1 n += 1 self.assertTrue(float(i) / n > 0.95) print("pattern.en.inflect.pluralize()")
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("horribles" => "horrible"). from pattern.db import Datasheet test = {} for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")): if tag == "j": test.setdefault(lemma, []).append(w) i, n = 0, 0 for pred, attr in test.items(): attr = sorted(attr, key=len, reverse=True)[0] if es.predicative(attr) == pred: i += 1 n += 1 self.assertTrue(float(i) / n > 0.92) print "pattern.es.predicative()"
def test_gender(self): # Assert the accuracy of the gender disambiguation algorithm. from pattern.db import Datasheet i, n = 0, 0 for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): g = it.gender(sg) if mf in g and it.PLURAL not in g: i += 1 g = it.gender(pl) if mf in g and it.PLURAL in g: i += 1 n += 2 self.assertTrue(float(i) / n > 0.92) print("pattern.it.gender()")
def test_pluralize(self): # Assert "octopodes" for classical plural of "octopus". # Assert "octopuses" for modern plural. self.assertEqual("octopodes", en.inflect.pluralize("octopus", classical=True)) self.assertEqual("octopuses", en.inflect.pluralize("octopus", classical=False)) # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")): if en.inflect.pluralize(sg) == pl: i +=1 n += 1 self.assertTrue(float(i) / n > 0.95) print "pattern.en.inflect.pluralize()"
def test_predicative(self): # Assert the accuracy of the predicative algorithm ("horribles" => "horrible"). from pattern.db import Datasheet test = {} for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")): if tag == "j": test.setdefault(lemma, []).append(w) i, n = 0, 0 for pred, attr in test.items(): attr = sorted(attr, key=len, reverse=True)[0] if es.predicative(attr) == pred: i += 1 n += 1 self.assertTrue(float(i) / n > 0.92) print("pattern.es.predicative()")
def test_singularize(self): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet test = {} for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")): if tag == "n": test.setdefault(lemma, []).append(w) i, n = 0, 0 for sg, pl in test.items(): pl = sorted(pl, key=len, reverse=True)[0] if es.singularize(pl) == sg: i += 1 n += 1 self.assertTrue(float(i) / n > 0.93) print("pattern.es.singularize()")
def classify(self, document): ''' This method is used to classify new documents. Uses the saved model. ''' #Loading csv predictions and corpora documents. try: nb_predictions = Datasheet.load("predictions/NB/patterns_nb.csv") nb_corpus = Datasheet.load("corpora/NB/nb.csv") index_pred = dict.fromkeys(nb_predictions.columns[0], True) index_corp = dict.fromkeys(nb_corpus.columns[0], True) except: nb_predictions = Datasheet() nb_corpus = Datasheet() index_pred = {} index_corp = {} #Load model from file system classifier = Classifier.load('models/nb_model.ept') label = classifier.classify(Document(document)) probability = classifier.classify(Document(document), discrete=False)[label] id = str(hash(label + document)) if ("positive" in label): if len(nb_predictions) == 0 or id not in index_pred: nb_predictions.append([id, label, document, probability]) index_pred[id] = True if len(nb_corpus) == 0 or id not in index_corp: nb_corpus.append([id, label, document, probability]) index_corp[id] = True nb_predictions.save("predictions/NB/patterns_nb.csv") nb_corpus.save("corpora/NB/nb.csv") return label
def classify(self, document): ''' This method is used to classify new documents. Uses the saved model. ''' #Loading csv predictions and corpora documents. try: svm_predictions = Datasheet.load("predictions/svm.csv") svm_corpus = Datasheet.load("corpora/svm/svm.csv") index_pred = dict.fromkeys(svm_predictions.columns[0], True) index_corp = dict.fromkeys(svm_corpus.columns[0], True) except: svm_predictions = Datasheet() svm_corpus = Datasheet() index_pred = {} index_corp = {} #Load model from file system classifier = Classifier.load('models/svm_model2.ept') label = classifier.classify(Document(document)) id = str(hash(label + document)) if ("positive" in label): if len(svm_predictions) == 0 or id not in index_pred: svm_predictions.append([id, label, document]) index_pred[id] = True if len(svm_corpus) == 0 or id not in index_corp: svm_corpus.append([id, label, document]) index_corp[id] = True svm_predictions.save("predictions/svm.csv") svm_corpus.save("corpora/svm/svm.csv") return label
def test_sentiment(self): # Assert < 0 for negative adjectives and > 0 for positive adjectives. self.assertTrue(nl.sentiment("geweldig")[0] > 0) self.assertTrue(nl.sentiment("verschrikkelijk")[0] < 0) # Assert the accuracy of the sentiment analysis. # Given are the scores for 3,000 book reviews. # The baseline should increase (not decrease) when the algorithm is modified. from pattern.db import Datasheet from pattern.metrics import test reviews = [] for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-nl-bol.com.csv")): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: nl.positive(review), reviews) self.assertTrue(A > 0.80) self.assertTrue(P > 0.77) self.assertTrue(R > 0.85) self.assertTrue(F > 0.81) print "pattern.nl.sentiment()"
def test_sentiment(self): # Assert < 0 for negative adjectives and > 0 for positive adjectives. self.assertTrue(fr.sentiment("fabuleux")[0] > 0) self.assertTrue(fr.sentiment("terrible")[0] < 0) # Assert the accuracy of the sentiment analysis. # Given are the scores for 1,500 book reviews. # The baseline should increase (not decrease) when the algorithm is modified. from pattern.db import Datasheet from pattern.metrics import test reviews = [] for review, score in Datasheet.load(os.path.join(PATH, "corpora", "polarity-fr-amazon.csv")): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: fr.positive(review), reviews) self.assertTrue(A > 0.75) self.assertTrue(P > 0.76) self.assertTrue(R > 0.73) self.assertTrue(F > 0.75) print "pattern.fr.sentiment()"
def test_sentiment(self): # Assert < 0 for negative adjectives and > 0 for positive adjectives. self.assertTrue(en.sentiment("wonderful")[0] > 0) self.assertTrue(en.sentiment("horrible")[0] < 0) self.assertTrue(en.sentiment(en.wordnet.synsets("horrible", pos="JJ")[0])[0] < 0) self.assertTrue(en.sentiment(en.Text(en.parse("A bad book. Really horrible.")))[0] < 0) # Assert the accuracy of the sentiment analysis. # Given are the scores for Pang & Lee's polarity dataset v2.0: # http://www.cs.cornell.edu/people/pabo/movie-review-data/ # The baseline should increase (not decrease) when the algorithm is modified. from pattern.db import Datasheet from pattern.metrics import test reviews = [] for score, review in Datasheet.load(os.path.join("corpora", "pang&lee-polarity.txt")): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: en.positive(review), reviews) self.assertTrue(A > 0.71) self.assertTrue(P > 0.72) self.assertTrue(R > 0.70) self.assertTrue(F > 0.71) print "pattern.en.sentiment()"