def test_sentiment(self):
# Assert < 0 for negative adjectives and > 0 for positive adjectives.
self.assertTrue(en.sentiment("wonderful")[0] > 0)
self.assertTrue(en.sentiment("horrible")[0] < 0)
self.assertTrue(en.sentiment(en.wordnet.synsets("horrible", pos="JJ")[0])[0] < 0)
self.assertTrue(en.sentiment(en.Text(en.parse("A bad book. Really horrible.")))[0] < 0)
# Assert that :) and :( are recognized.
self.assertTrue(en.sentiment(":)")[0] > 0)
self.assertTrue(en.sentiment(":(")[0] < 0)
# Assert the accuracy of the sentiment analysis (for the positive class).
# Given are the scores for Pang & Lee's polarity dataset v2.0:
# http://www.cs.cornell.edu/people/pabo/movie-review-data/
# The baseline should increase (not decrease) when the algorithm is modified.
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee1.csv")):
reviews.append((review, int(score) > 0))
A, P, R, F = test(lambda review: en.positive(review), reviews)
self.assertTrue(A > 0.755)
self.assertTrue(P > 0.760)
self.assertTrue(R > 0.747)
self.assertTrue(F > 0.754)
# Assert the accuracy of the sentiment analysis on short text (for the positive class).
# Given are the scores for Pang & Lee's sentence polarity dataset v1.0:
# http://www.cs.cornell.edu/people/pabo/movie-review-data/
reviews = []
for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee2.csv")):
reviews.append((review, int(score) > 0))
A, P, R, F = test(lambda review: en.positive(review), reviews)
self.assertTrue(A > 0.642)
self.assertTrue(P > 0.653)
self.assertTrue(R > 0.607)
self.assertTrue(F > 0.629)
print "pattern.en.sentiment()"
def testModel(self, *args):
''' Perform learning of a Model from training data.
'''
documents = []
data = Datasheet.load(os.path.join("corpora","twitter","trainer", "tweets_stream_data_nb.csv"))
data2 = Datasheet.load(os.path.join("corpora","twitter","trainer", "tweets_stream_data_svm.csv"))
data3 = Datasheet.load(os.path.join("corpora","twitter","trainer", "ensenmble.csv"))
if args:
classifier = Classifier.load('models/nb_model.ept')
print "Document class is %s" % classifier.classify(Document(args[0]))
print "Document probability is : ", classifier.classify(Document(args[0]), discrete=False)
label = classifier.classify(Document(args[0]), discrete=False)
print label["positive"]
else:
i = n = 0
pos=neg=0
classifier = Classifier.load('models/nb_model.ept')
data = shuffled(data)
for document, label in data[:]+data2[:]+data3[:]:
doc_vector = Document(document, type=str(label), stopwords=True)
documents.append(doc_vector)
if 'positive' in label:
pos+=1
else:
neg+=1
print "10-fold CV"
print k_fold_cv(NB, documents=documents, folds=10)
print "Neg: %s, Pos: %s" % (neg, pos)
print classifier.distribution
print "Classes in Naive Bayes Classifier"
print classifier.classes
print "Area Under the Curve: %0.6f" % classifier.auc(documents, k=10)
print "Model Performance (Positive Classifications)"
accuracy, precision, recall, f1 = classifier.test(data[:]+data2[:]+data3[:], target='positive')
print "Accuracy = %.6f; F-Score = %.6f; Precision = %.6f; Recall = %.6f" % (accuracy, f1, precision, recall)
print "Model Performance(Negative Classifications)"
accuracy, precision, recall, f1 = classifier.test(data[:]+data2[:]+data3[:], target='negative')
print "Accuracy = %.6f; F-Score = %.6f; Precision = %.6f; Recall = %.6f" % (accuracy, f1, precision, recall)
print "Model Performance"
accuracy, precision, recall, f1 = classifier.test(data[:]+data2[:]+data3[:])
print "Accuracy = %.6f; F-Score = %.6f; Precision = %.6f; Recall = %.6f" % (accuracy, f1, precision, recall)
print "Confusion Matrix"
print classifier.confusion_matrix(data[:]+data2[:]+data3[:])
print classifier.confusion_matrix(data[:]+data2[:]+data3[:])('positive')
print classifier.confusion_matrix(data[:]+data2[:]+data3[:])('negative')
def train(self, train_path):
""" Train classifier on features from headline and article text """
if self.debug:
tick = time()
logging.info("Training new model with %s" % (train_path,))
logging.info("Loading/shuffling training data...")
train_data_1 = Datasheet.load(train_path)
shuffle(train_data_1)
train_texts_1 = zip(train_data_1.columns[0], train_data_1.columns[1])
train_labels_1 = [0 if x == '0' else 1 for x in train_data_1.columns[-1]]
if self.debug:
logging.info('Fitting training data')
pipeline_1 = self.create_pipeline()
pipeline_1.fit(train_texts_1, train_labels_1)
if self.debug:
logging.info("Done in %0.2fs" % (time() - tick,))
train_data_2 = Datasheet()
for row in train_data_1.rows:
if row[-1] != '0':
train_data_2.append(row)
train_texts_2 = zip(train_data_2.columns[0], train_data_2.columns[1])
train_labels_2 = train_data_2.columns[-1]
pipeline_2 = self.create_pipeline()
pipeline_2.fit(train_texts_2, train_labels_2)
return pipeline_1, pipeline_2
def test_sentiment_twitter(self):
sanders = os.path.join(PATH, "corpora", "polarity-en-sanders.csv")
if os.path.exists(sanders):
# Assert the accuracy of the sentiment analysis on tweets.
# Given are the scores for Sanders Twitter Sentiment Corpus:
# http://www.sananalytics.com/lab/twitter-sentiment/
# Positive + neutral is taken as polarity >= 0.0,
# Negative is taken as polarity < 0.0.
# Since there are a lot of neutral cases,
# and the algorithm predicts 0.0 by default (i.e., majority class) the results are good.
# Distinguishing negative from neutral from positive is a much
# harder task
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for i, id, date, tweet, polarity, topic in Datasheet.load(sanders):
if polarity != "irrelevant":
reviews.append(
(tweet, polarity in ("positive", "neutral")))
A, P, R, F = test(
lambda review: en.positive(review, threshold=0.0), reviews)
#print(A, P, R, F)
self.assertTrue(A > 0.824)
self.assertTrue(P > 0.879)
self.assertTrue(R > 0.911)
self.assertTrue(F > 0.895)
def test_sentiment(self):
# Assert < 0 for negative adjectives and > 0 for positive adjectives.
self.assertTrue(en.sentiment("wonderful")[0] > 0)
self.assertTrue(en.sentiment("horrible")[0] < 0)
self.assertTrue(
en.sentiment(en.wordnet.synsets("horrible", pos="JJ")[0])[0] < 0)
self.assertTrue(
en.sentiment(en.Text(en.parse("A bad book. Really horrible.")))[0]
< 0)
# Assert the accuracy of the sentiment analysis.
# Given are the scores for Pang & Lee's polarity dataset v2.0:
# http://www.cs.cornell.edu/people/pabo/movie-review-data/
# The baseline should increase (not decrease) when the algorithm is modified.
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for score, review in Datasheet.load(
os.path.join("corpora", "pang&lee-polarity.txt")):
reviews.append((review, int(score) > 0))
A, P, R, F = test(lambda review: en.positive(review), reviews)
self.assertTrue(A > 0.71)
self.assertTrue(P > 0.72)
self.assertTrue(R > 0.70)
self.assertTrue(F > 0.71)
print "pattern.en.sentiment()"
def test_spelling(self):
# Assert case-sensitivity + numbers.
for a, b in (
(".", "."),
("?", "?"),
("!", "!"),
("I", "I"),
("a", "a"),
("42", "42"),
("3.14", "3.14"),
("The", "The"),
("the", "the")):
self.assertEqual(en.suggest(a)[0][0], b)
# Assert spelling suggestion accuracy.
# Note: simply training on more text will not improve accuracy.
i = j = 0.0
from pattern.db import Datasheet
for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv")):
for w in wrong.split(" "):
if en.suggest(w)[0][0] == correct:
i += 1
else:
j += 1
self.assertTrue(i / (i + j) > 0.70)
print("pattern.en.suggest()")
def test_modality(self):
# Assert -1.0 => +1.0 representing the degree of certainty.
v = en.modality(en.Sentence(en.parse("I wish it would stop raining.")))
self.assertTrue(v < 0)
v = en.modality(
en.Sentence(en.parse("It will surely stop raining soon.")))
self.assertTrue(v > 0)
# Assert the accuracy of the modality algorithm.
# Given are the scores for the CoNLL-2010 Shared Task 1 Wikipedia uncertainty data:
# http://www.inf.u-szeged.hu/rgai/conll2010st/tasks.html#task1
# The baseline should increase (not decrease) when the algorithm is modified.
from pattern.db import Datasheet
from pattern.metrics import test
sentences = []
for certain, sentence in Datasheet.load(
os.path.join(PATH, "corpora", "uncertainty-conll2010.csv")):
sentence = en.parse(sentence, chunks=False, light=True)
sentence = en.Sentence(sentence)
sentences.append((sentence, int(certain) > 0))
A, P, R, F = test(lambda sentence: en.modality(sentence) > 0.5,
sentences)
#print A, P, R, F
self.assertTrue(A > 0.69)
self.assertTrue(P > 0.71)
self.assertTrue(R > 0.64)
self.assertTrue(F > 0.67)
print "pattern.en.modality()"
def test_sentiment_twitter(self):
sanders = os.path.join(PATH, "corpora", "polarity-en-sanders.csv")
if os.path.exists(sanders):
# Assert the accuracy of the sentiment analysis on tweets.
# Given are the scores for Sanders Twitter Sentiment Corpus:
# http://www.sananalytics.com/lab/twitter-sentiment/
# Positive + neutral is taken as polarity >= 0.0,
# Negative is taken as polarity < 0.0.
# Since there are a lot of neutral cases,
# and the algorithm predicts 0.0 by default (i.e., majority class) the results are good.
# Distinguishing negative from neutral from positive is a much harder task
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for i, id, date, tweet, polarity, topic in Datasheet.load(sanders):
if polarity != "irrelevant":
reviews.append((tweet, polarity
in ("positive", "neutral")))
A, P, R, F = test(
lambda review: en.positive(review, threshold=0.0), reviews)
#print A, P, R, F
self.assertTrue(A > 0.824)
self.assertTrue(P > 0.879)
self.assertTrue(R > 0.911)
self.assertTrue(F > 0.895)
def test_spelling(self):
# Assert case-sensitivity + numbers.
for a, b in (
( ".", "." ),
( "?", "?" ),
( "!", "!" ),
( "I", "I" ),
( "a", "a" ),
( "42", "42" ),
("3.14", "3.14"),
( "The", "The" ),
( "the", "the" )):
self.assertEqual(en.suggest(a)[0][0], b)
# Assert spelling suggestion accuracy.
# Note: simply training on more text will not improve accuracy.
i = j = 0.0
from pattern.db import Datasheet
for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv")):
for w in wrong.split(" "):
if en.suggest(w)[0][0] == correct:
i += 1
else:
j += 1
self.assertTrue(i / (i + j) > 0.70)
print("pattern.en.suggest()")
def load_domains(self):
sources_path = pd('data', 'source_data.csv')
domain_file = Datasheet.load(sources_path, headers=True)
for row in domain_file:
url = row[1]
cats = row[2:]
self.cat_dict[url] = cats
def test_modality(self):
# Assert -1.0 => +1.0 representing the degree of certainty.
v = en.modality(en.Sentence(en.parse("I wish it would stop raining.")))
self.assertTrue(v < 0)
v = en.modality(
en.Sentence(en.parse("It will surely stop raining soon.")))
self.assertTrue(v > 0)
# Assert the accuracy of the modality algorithm.
# Given are the scores for the CoNLL-2010 Shared Task 1 Wikipedia uncertainty data:
# http://www.inf.u-szeged.hu/rgai/conll2010st/tasks.html#task1
# The baseline should increase (not decrease) when the algorithm is
# modified.
from pattern.db import Datasheet
from pattern.metrics import test
sentences = []
for certain, sentence in Datasheet.load(os.path.join(PATH, "corpora", "uncertainty-conll2010.csv")):
sentence = en.parse(sentence, chunks=False, light=True)
sentence = en.Sentence(sentence)
sentences.append((sentence, int(certain) > 0))
A, P, R, F = test(
lambda sentence: en.modality(sentence) > 0.5, sentences)
#print(A, P, R, F)
self.assertTrue(A > 0.69)
self.assertTrue(P > 0.72)
self.assertTrue(R > 0.64)
self.assertTrue(F > 0.68)
print("pattern.en.modality()")
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")):
if en.inflect.singularize(pl) == sg:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.95)
print "pattern.en.inflect.singularize()"
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
if it.singularize(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.84)
print "pattern.it.singularize()"
def model(top=None):
""" Returns a Model of e-mail messages.
Document type=True => HAM, False => SPAM.
Documents are mostly of a technical nature (developer forum posts).
"""
documents = []
for score, message in Datasheet.load(os.path.join(PATH, "corpora", "spam-apache.csv")):
document = vector.Document(message, stemmer="porter", top=top, type=int(score) > 0)
documents.append(document)
return vector.Model(documents)
def model(top=None):
""" Returns a Model of e-mail messages.
Document type=True => HAM, False => SPAM.
Documents are mostly of a technical nature (developer forum posts).
"""
documents = []
for score, message in Datasheet.load(os.path.join(PATH, "corpora", "spam-apache.csv")):
document = vector.Document(message, stemmer="porter", top=top, type=int(score) > 0)
documents.append(document)
return vector.Model(documents)
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
if it.singularize(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.84)
print("pattern.it.singularize()")
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")):
if nl.singularize(pl) == sg:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.88)
print "pattern.nl.singularize()"
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("felle" => "fel").
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")):
if nl.predicative(attr) == pred:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.96)
print "pattern.nl.predicative()"
def test_attributive(self):
# Assert the accuracy of the attributive algorithm ("fel" => "felle").
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, sg, pl in Datasheet.load(os.path.join("corpora", "celex-wordforms-nl.csv")):
if nl.attributive(pred) == attr:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.96)
print "pattern.nl.attributive()"
def test_pluralize(self):
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
if it.pluralize(sg) == pl:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.93)
print("pattern.it.pluralize()")
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")):
if en.inflect.singularize(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.95)
print("pattern.en.inflect.singularize()")
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")):
if nl.singularize(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.88)
print("pattern.nl.singularize()")
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("felle" => "fel").
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")):
if nl.predicative(attr) == pred:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.96)
print("pattern.nl.predicative()")
def main():
logging.basicConfig(level=logging.INFO)
argparser = ArgumentParser(description=__doc__)
argparser.add_argument("-t",
"--trainset",
action="store",
default=None,
help=("Path to training data "
"[default: %(default)s]"))
argparser.add_argument("-m",
"--model",
action="store",
help="Path to model")
argparser.add_argument("-d",
"--dump",
action="store_true",
help="Pickle trained model? [default: False]")
argparser.add_argument("-v",
"--verbose",
action="store_true",
default=False,
help="Verbose [default: quiet]")
argparser.add_argument("-c",
"--classify",
action="store",
default=None,
help=("Path to data to classify "
"[default: %(default)s]"))
argparser.add_argument("-s",
"--save",
action="store",
default='output.csv',
help=("Path to output file"
"[default = output.csv]"))
args = argparser.parse_args()
clf = SensationalismClassifier(train_data=args.trainset,
model=args.model,
dump=args.dump,
debug=args.verbose)
if args.classify:
OUTPUT_PATH = args.save
if clf.debug:
tick = time()
to_classify = Datasheet.load(args.classify)
classified_data = clf.classify(to_classify)
output = Datasheet(classified_data)
output.save(pd(OUTPUT_PATH))
if clf.debug:
sys.stderr.write("\nProcessed %d items in %0.2fs" %
(len(classified_data), time() - tick))
def scrape_news_text(news_url):
global counter
news_html = requests.get(news_url).content
# print(news_html)
'''convert html to BeautifulSoup object'''
news_soup = BeautifulSoup(news_html, 'lxml')
# soup.find("div", {"id": "articlebody"})
# paragraphs = [par.text for par in news_soup.find_all('p')]
# news_text = '\n'.join(paragraphs)
# print(news_soup.find("div", {"id": "articleText"}))
date_object = news_soup.find(itemprop="datePublished")
news_object = news_soup.find("div", {"id": "articleText"})
if date_object is None:
return " "
if news_object is None:
return " "
news_date = date_object.get_text(
) # find("div", {"id": "articleText"}).text
news_text = news_object.text
# print(news_date)
# print(news_text)
print(news_url)
try:
# We'll store tweets in a Datasheet.
# A Datasheet is a table of rows and columns that can be exported as a CSV-file.
# In the first column, we'll store a unique id for each tweet.
# We only want to add the latest tweets, i.e., those we haven't seen yet.
# With an index on the first column we can quickly check if an id already exists.
# The pd() function returns the parent directory of this script + any given path.
table = Datasheet.load(pd("nasdaq2.csv"))
except:
table = Datasheet()
news_sentiment = sentiment(news_text)
print(news_sentiment)
table.append([counter, news_date, news_url, news_sentiment])
table.save(pd("nasdaq2.csv"))
counter += 1
return news_text
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for tag, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "celex-wordforms-de.csv")):
if tag == "n":
if de.singularize(pl) == sg:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.81)
print "pattern.de.singularize()"
def test_pluralize(self):
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for tag, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-de-celex.csv")):
if tag == "n":
if de.pluralize(sg) == pl:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.69)
print "pattern.de.pluralize()"
def test_intertextuality(self):
# Evaluate accuracy for plagiarism detection.
from pattern.db import Datasheet
data = Datasheet.load(os.path.join(PATH, "corpora", "plagiarism-clough&stevenson.csv"))
data = [((txt, src), int(plagiarism) > 0) for txt, src, plagiarism in data]
def plagiarism(txt, src):
return metrics.intertextuality([txt, src], n=3)[0,1] > 0.05
A, P, R, F = metrics.test(lambda x: plagiarism(*x), data)
self.assertTrue(P > 0.96)
self.assertTrue(R > 0.94)
print "pattern.metrics.intertextuality()"
def test_spelling(self):
# Assert spelling suggestion accuracy.
i = j = 0.0
from pattern.db import Datasheet
for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "birkbeck-spelling.csv")):
for w in wrong.split(" "):
if en.spelling(w)[0][0] == correct:
i += 1
else:
j += 1
self.assertTrue(i / (i+j) > 0.70)
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("belles" => "beau").
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, tag in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-fr-lexique.csv")):
if tag == "a":
if fr.predicative(attr) == pred:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.95)
print "pattern.fr.predicative()"
def test_intertextuality(self):
# Evaluate accuracy for plagiarism detection.
from pattern.db import Datasheet
data = Datasheet.load(os.path.join(PATH, "corpora", "plagiarism-clough&stevenson.csv"))
data = [((txt, src), int(plagiarism) > 0) for txt, src, plagiarism in data]
def plagiarism(txt, src):
return metrics.intertextuality([txt, src], n=3)[0,1] > 0.05
A, P, R, F = metrics.test(lambda x: plagiarism(*x), data)
self.assertTrue(P > 0.96)
self.assertTrue(R > 0.94)
print("pattern.metrics.intertextuality()")
def test_pluralize(self):
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(
os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
if it.pluralize(sg) == pl:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.93)
print "pattern.it.pluralize()"
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("großer" => "groß").
from pattern.db import Datasheet
i, n = 0, 0
for tag, pred, attr in Datasheet.load(os.path.join(PATH, "corpora", "celex-wordforms-de.csv")):
if tag == "a":
if de.predicative(attr) == pred:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.98)
print "pattern.de.predicative()"
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("großer" => "groß").
from pattern.db import Datasheet
i, n = 0, 0
for tag, pred, attr in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-de-celex.csv")):
if tag == "a":
if de.predicative(attr) == pred:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.98)
print("pattern.de.predicative()")
def test_pluralize(self):
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for tag, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-de-celex.csv")):
if tag == "n":
if de.pluralize(sg) == pl:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.69)
print("pattern.de.pluralize()")
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("belles" => "beau").
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, tag in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-fr-lexique.csv")):
if tag == "a":
if fr.predicative(attr) == pred:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.95)
print("pattern.fr.predicative()")
def load_domains(self):
"""loads domain information"""
sources_path = pd('data', 'source_data.csv')
domain_file = Datasheet.load(sources_path, headers=True)
for row in domain_file:
url = row[1]
if str(row[-1]).find("\""):
cats = row[2:-1]
else:
cats = row[2:]
self.cat_dict[url] = cats
def test_pluralize(self):
# Assert "auto's" as plural of "auto".
self.assertEqual("auto's", nl.inflect.pluralize("auto"))
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")):
if nl.pluralize(sg) == pl:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.74)
print("pattern.nl.pluralize()")
def test_pluralize(self):
# Assert "auto's" as plural of "auto".
self.assertEqual("auto's", nl.inflect.pluralize("auto"))
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")):
if nl.pluralize(sg) == pl:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.74)
print "pattern.nl.pluralize()"
def test_spelling(self):
# Assert spelling suggestion accuracy.
# Note: simply training on more text will not improve accuracy.
i = j = 0.0
from pattern.db import Datasheet
for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "birkbeck-spelling.csv")):
for w in wrong.split(" "):
if en.spelling(w)[0][0] == correct:
i += 1
else:
j += 1
self.assertTrue(i / (i+j) > 0.70)
def test_spelling(self):
# Assert spelling suggestion accuracy.
# Note: simply training on more text will not improve accuracy.
i = j = 0.0
from pattern.db import Datasheet
for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv")):
for w in wrong.split(" "):
if en.spelling(w)[0][0] == correct:
i += 1
else:
j += 1
self.assertTrue(i / (i+j) > 0.70)
def test_spelling(self):
i = j = 0.0
from pattern.db import Datasheet
for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-ru.csv")):
for w in wrong.split(" "):
suggested = ru.suggest(w)
if suggested[0][0] == correct:
i += 1
else:
j += 1
self.assertTrue(i / (i + j) > 0.65)
print("pattern.ru.suggest()")
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("cruciali" => "cruciale").
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
if pos != "j":
continue
if it.predicative(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.87)
print "pattern.it.predicative()"
def test_gender(self):
# Assert the accuracy of the gender disambiguation algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
g = it.gender(sg)
if mf in g and it.PLURAL not in g:
i += 1
g = it.gender(pl)
if mf in g and it.PLURAL in g:
i += 1
n += 2
self.assertTrue(float(i) / n > 0.92)
print "pattern.it.gender()"
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("cruciali" => "cruciale").
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(
os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
if pos != "j":
continue
if it.predicative(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.87)
print("pattern.it.predicative()")
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
test = {}
for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")):
if tag == "n": test.setdefault(lemma, []).append(w)
i, n = 0, 0
for sg, pl in test.items():
pl = sorted(pl, key=len, reverse=True)[0]
if es.singularize(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.93)
print "pattern.es.singularize()"
def test_pluralize(self):
# Assert "octopodes" for classical plural of "octopus".
# Assert "octopuses" for modern plural.
self.assertEqual("octopodes", en.inflect.pluralize("octopus", classical=True))
self.assertEqual("octopuses", en.inflect.pluralize("octopus", classical=False))
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")):
if en.inflect.pluralize(sg) == pl:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.95)
print("pattern.en.inflect.pluralize()")
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("horribles" => "horrible").
from pattern.db import Datasheet
test = {}
for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")):
if tag == "j": test.setdefault(lemma, []).append(w)
i, n = 0, 0
for pred, attr in test.items():
attr = sorted(attr, key=len, reverse=True)[0]
if es.predicative(attr) == pred:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.92)
print "pattern.es.predicative()"
def test_gender(self):
# Assert the accuracy of the gender disambiguation algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
g = it.gender(sg)
if mf in g and it.PLURAL not in g:
i += 1
g = it.gender(pl)
if mf in g and it.PLURAL in g:
i += 1
n += 2
self.assertTrue(float(i) / n > 0.92)
print("pattern.it.gender()")
def test_pluralize(self):
# Assert "octopodes" for classical plural of "octopus".
# Assert "octopuses" for modern plural.
self.assertEqual("octopodes", en.inflect.pluralize("octopus", classical=True))
self.assertEqual("octopuses", en.inflect.pluralize("octopus", classical=False))
# Assert the accuracy of the pluralization algorithm.
from pattern.db import Datasheet
i, n = 0, 0
for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")):
if en.inflect.pluralize(sg) == pl:
i +=1
n += 1
self.assertTrue(float(i) / n > 0.95)
print "pattern.en.inflect.pluralize()"
def test_predicative(self):
# Assert the accuracy of the predicative algorithm ("horribles" => "horrible").
from pattern.db import Datasheet
test = {}
for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")):
if tag == "j": test.setdefault(lemma, []).append(w)
i, n = 0, 0
for pred, attr in test.items():
attr = sorted(attr, key=len, reverse=True)[0]
if es.predicative(attr) == pred:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.92)
print("pattern.es.predicative()")
def test_singularize(self):
# Assert the accuracy of the singularization algorithm.
from pattern.db import Datasheet
test = {}
for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")):
if tag == "n": test.setdefault(lemma, []).append(w)
i, n = 0, 0
for sg, pl in test.items():
pl = sorted(pl, key=len, reverse=True)[0]
if es.singularize(pl) == sg:
i += 1
n += 1
self.assertTrue(float(i) / n > 0.93)
print("pattern.es.singularize()")
def classify(self, document):
''' This method is used to classify new documents. Uses the saved model.
'''
#Loading csv predictions and corpora documents.
try:
nb_predictions = Datasheet.load("predictions/NB/patterns_nb.csv")
nb_corpus = Datasheet.load("corpora/NB/nb.csv")
index_pred = dict.fromkeys(nb_predictions.columns[0], True)
index_corp = dict.fromkeys(nb_corpus.columns[0], True)
except:
nb_predictions = Datasheet()
nb_corpus = Datasheet()
index_pred = {}
index_corp = {}
#Load model from file system
classifier = Classifier.load('models/nb_model.ept')
label = classifier.classify(Document(document))
probability = classifier.classify(Document(document), discrete=False)[label]
id = str(hash(label + document))
if ("positive" in label):
if len(nb_predictions) == 0 or id not in index_pred:
nb_predictions.append([id, label, document, probability])
index_pred[id] = True
if len(nb_corpus) == 0 or id not in index_corp:
nb_corpus.append([id, label, document, probability])
index_corp[id] = True
nb_predictions.save("predictions/NB/patterns_nb.csv")
nb_corpus.save("corpora/NB/nb.csv")
return label
def classify(self, document):
''' This method is used to classify new documents. Uses the saved model.
'''
#Loading csv predictions and corpora documents.
try:
svm_predictions = Datasheet.load("predictions/svm.csv")
svm_corpus = Datasheet.load("corpora/svm/svm.csv")
index_pred = dict.fromkeys(svm_predictions.columns[0], True)
index_corp = dict.fromkeys(svm_corpus.columns[0], True)
except:
svm_predictions = Datasheet()
svm_corpus = Datasheet()
index_pred = {}
index_corp = {}
#Load model from file system
classifier = Classifier.load('models/svm_model2.ept')
label = classifier.classify(Document(document))
id = str(hash(label + document))
if ("positive" in label):
if len(svm_predictions) == 0 or id not in index_pred:
svm_predictions.append([id, label, document])
index_pred[id] = True
if len(svm_corpus) == 0 or id not in index_corp:
svm_corpus.append([id, label, document])
index_corp[id] = True
svm_predictions.save("predictions/svm.csv")
svm_corpus.save("corpora/svm/svm.csv")
return label
def test_sentiment(self):
# Assert < 0 for negative adjectives and > 0 for positive adjectives.
self.assertTrue(nl.sentiment("geweldig")[0] > 0)
self.assertTrue(nl.sentiment("verschrikkelijk")[0] < 0)
# Assert the accuracy of the sentiment analysis.
# Given are the scores for 3,000 book reviews.
# The baseline should increase (not decrease) when the algorithm is modified.
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-nl-bol.com.csv")):
reviews.append((review, int(score) > 0))
A, P, R, F = test(lambda review: nl.positive(review), reviews)
self.assertTrue(A > 0.80)
self.assertTrue(P > 0.77)
self.assertTrue(R > 0.85)
self.assertTrue(F > 0.81)
print "pattern.nl.sentiment()"
def test_sentiment(self):
# Assert < 0 for negative adjectives and > 0 for positive adjectives.
self.assertTrue(fr.sentiment("fabuleux")[0] > 0)
self.assertTrue(fr.sentiment("terrible")[0] < 0)
# Assert the accuracy of the sentiment analysis.
# Given are the scores for 1,500 book reviews.
# The baseline should increase (not decrease) when the algorithm is modified.
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for review, score in Datasheet.load(os.path.join(PATH, "corpora", "polarity-fr-amazon.csv")):
reviews.append((review, int(score) > 0))
A, P, R, F = test(lambda review: fr.positive(review), reviews)
self.assertTrue(A > 0.75)
self.assertTrue(P > 0.76)
self.assertTrue(R > 0.73)
self.assertTrue(F > 0.75)
print "pattern.fr.sentiment()"
def test_sentiment(self):
# Assert < 0 for negative adjectives and > 0 for positive adjectives.
self.assertTrue(nl.sentiment("geweldig")[0] > 0)
self.assertTrue(nl.sentiment("verschrikkelijk")[0] < 0)
# Assert the accuracy of the sentiment analysis.
# Given are the scores for 3,000 book reviews.
# The baseline should increase (not decrease) when the algorithm is modified.
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-nl-bol.com.csv")):
reviews.append((review, int(score) > 0))
A, P, R, F = test(lambda review: nl.positive(review), reviews)
self.assertTrue(A > 0.80)
self.assertTrue(P > 0.77)
self.assertTrue(R > 0.85)
self.assertTrue(F > 0.81)
print "pattern.nl.sentiment()"
def test_sentiment(self):
# Assert < 0 for negative adjectives and > 0 for positive adjectives.
self.assertTrue(en.sentiment("wonderful")[0] > 0)
self.assertTrue(en.sentiment("horrible")[0] < 0)
self.assertTrue(en.sentiment(en.wordnet.synsets("horrible", pos="JJ")[0])[0] < 0)
self.assertTrue(en.sentiment(en.Text(en.parse("A bad book. Really horrible.")))[0] < 0)
# Assert the accuracy of the sentiment analysis.
# Given are the scores for Pang & Lee's polarity dataset v2.0:
# http://www.cs.cornell.edu/people/pabo/movie-review-data/
# The baseline should increase (not decrease) when the algorithm is modified.
from pattern.db import Datasheet
from pattern.metrics import test
reviews = []
for score, review in Datasheet.load(os.path.join("corpora", "pang&lee-polarity.txt")):
reviews.append((review, int(score) > 0))
A, P, R, F = test(lambda review: en.positive(review), reviews)
self.assertTrue(A > 0.71)
self.assertTrue(P > 0.72)
self.assertTrue(R > 0.70)
self.assertTrue(F > 0.71)
print "pattern.en.sentiment()"
