def setups():
"""set up classifier if not yet trained"""
if philosophyfilter.is_ready():
return
db.close()
db.connection(db='test_opp')
ham = Doc(url='http://umsu.de/papers/magnetism2.pdf')
ham.load_from_db()
ham.content = readfile(os.path.join(testdir, 'attitudes.txt'))
ham.update_db()
spam = Doc(url='http://umsu.de/papers/spam.pdf')
spam.load_from_db()
spam.content = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
enim ad minim veniam, quis nostrud exercitation ullamco laboris
nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor
in reprehenderit in voluptate velit esse cillum dolore eu
fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est
laborum.
"""
spam.update_db()
cur = db.cursor()
query = "SELECT cat_id FROM cats WHERE label=%s LIMIT 1"
cur.execute(query, ('philosophy',))
cat_id = cur.fetchall()[0]
query = ("INSERT IGNORE INTO docs2cats (doc_id, cat_id, strength, is_training)"
"VALUES (%s, %s, %s, %s)")
cur.execute(query, (ham.doc_id, cat_id, 1, 1))
cur.execute(query, (spam.doc_id, cat_id, 0, 1))
philosophyfilter.update()
def test_train():
mc = DocClassifier(picklefile)
doc = Doc(url='http://umsu.de/papers/variations.pdf')
doc.content = readfile(os.path.join(testdir, 'attitudes.txt'))
mc.train([doc], [True])
mc.save()
assert True
def test_get_duplicate(testdb):
doc = Doc(url='http://umsu.de/papers/driver-2011.pdf')
doc.link = Link(url='http://umsu.de/papers/driver-2011.pdf')
doc.content = readfile(os.path.join(testdir, 'attitudes.txt'))
doc.numwords = 13940
doc.numpages = 26
doc.authors = 'Wolfang Schwarz'
doc.title = 'Lost memories and useless coins: Revisiting the absentminded driver'
doc.update_db()
doc2 = Doc(url='http://download.springer.com/static/pdf/307/art%253A10.1007%252Fs11229-015-0699-z.pdf')
doc2.link = Link(url=doc2.url)
doc2.content = 'abcdefghjik'+readfile(os.path.join(testdir, 'attitudes.txt'))
doc2.numwords = 14130
doc2.numpages = 29
doc2.authors = 'Wolfang Schwarz'
doc2.title = 'Lost memories and useless coins: revisiting the absentminded driver'
dupe = scraper.get_duplicate(doc2)
assert dupe.doc_id == doc.doc_id
def test_cv():
doc = Doc(url="http://umsu.de/papers/cv.pdf")
doc.link = Link(url="http://umsu.de/papers/cv.pdf")
doc.link.anchortext = "CV"
doc.link.context = "CV"
doc.content = readfile(os.path.join(testdir, "cv.txt"))
doc.numwords = 10200
doc.numpages = 22
doc.meta_confidence = 92
assert paperfilter.evaluate(doc) < 0.4
def test_gooddoc_badlink():
doc = Doc(url="http://umsu.de/papers/variations.pdf")
doc.link = Link(url="http://umsu.de/papers/variations.pdf")
doc.link.anchortext = "slides"
doc.link.context = "The slides for my talk"
doc.content = readfile(os.path.join(testdir, "attitudes.txt"))
doc.numwords = 10200
doc.numpages = 22
doc.meta_confidence = 92
assert paperfilter.evaluate(doc) < 0.8
def test_gooddoc():
doc = Doc(url="http://umsu.de/papers/variations.pdf")
doc.link = Link(url="http://umsu.de/papers/variations.pdf")
doc.link.anchortext = "Download"
doc.link.context = "Foo bar"
doc.content = readfile(os.path.join(testdir, "attitudes.txt"))
doc.numwords = 10200
doc.numpages = 22
doc.meta_confidence = 92
assert paperfilter.evaluate(doc) > 0.98
def test_classify():
mc = DocClassifier(picklefile)
ham = Doc(url='http://umsu.de/papers/variations.pdf')
ham.content = readfile(os.path.join(testdir, 'attitudes.txt'))
spam = Doc(url='http://umsu.de/papers/spam.pdf')
spam.content = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
enim ad minim veniam, quis nostrud exercitation ullamco laboris
nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor
in reprehenderit in voluptate velit esse cillum dolore eu
fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est
laborum.
"""
spam.content *= 50
mc.train([ham, spam], [True, False])
ham.content += 'foo bar'
prob = mc.classify(ham)
assert prob > 0.5
def test_gooddoc(setups):
doc = Doc(url='http://umsu.de/papers/variations.pdf')
doc.content = readfile(os.path.join(testdir, 'attitudes.txt'))
assert philosophyfilter.evaluate(doc) > 0.6