示例#1
0
def setups():
    """set up classifier if not yet trained"""
    if philosophyfilter.is_ready():
        return
    db.close()
    db.connection(db='test_opp')
    ham = Doc(url='http://umsu.de/papers/magnetism2.pdf')
    ham.load_from_db()
    ham.content = readfile(os.path.join(testdir, 'attitudes.txt'))
    ham.update_db()
    spam = Doc(url='http://umsu.de/papers/spam.pdf')
    spam.load_from_db()
    spam.content = """ 
       Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
       eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
       enim ad minim veniam, quis nostrud exercitation ullamco laboris
       nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor
       in reprehenderit in voluptate velit esse cillum dolore eu
       fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
       proident, sunt in culpa qui officia deserunt mollit anim id est
       laborum. 
    """
    spam.update_db()
    cur = db.cursor()
    query = "SELECT cat_id FROM cats WHERE label=%s LIMIT 1"
    cur.execute(query, ('philosophy',))
    cat_id = cur.fetchall()[0]
    query = ("INSERT IGNORE INTO docs2cats (doc_id, cat_id, strength, is_training)"
             "VALUES (%s, %s, %s, %s)")
    cur.execute(query, (ham.doc_id, cat_id, 1, 1))
    cur.execute(query, (spam.doc_id, cat_id, 0, 1))
    philosophyfilter.update()
示例#2
0
def test_train():
    mc = DocClassifier(picklefile)
    doc = Doc(url='http://umsu.de/papers/variations.pdf')
    doc.content = readfile(os.path.join(testdir, 'attitudes.txt'))
    mc.train([doc], [True])
    mc.save()
    assert True
示例#3
0
def test_get_duplicate(testdb):
    doc = Doc(url='http://umsu.de/papers/driver-2011.pdf')
    doc.link = Link(url='http://umsu.de/papers/driver-2011.pdf')
    doc.content = readfile(os.path.join(testdir, 'attitudes.txt'))
    doc.numwords = 13940
    doc.numpages = 26
    doc.authors = 'Wolfang Schwarz'
    doc.title = 'Lost memories and useless coins: Revisiting the absentminded driver'
    doc.update_db()
    doc2 = Doc(url='http://download.springer.com/static/pdf/307/art%253A10.1007%252Fs11229-015-0699-z.pdf')
    doc2.link = Link(url=doc2.url)
    doc2.content = 'abcdefghjik'+readfile(os.path.join(testdir, 'attitudes.txt'))
    doc2.numwords = 14130
    doc2.numpages = 29
    doc2.authors = 'Wolfang Schwarz'
    doc2.title = 'Lost memories and useless coins: revisiting the absentminded driver'
    dupe = scraper.get_duplicate(doc2)
    assert dupe.doc_id == doc.doc_id
示例#4
0
def test_cv():
    doc = Doc(url="http://umsu.de/papers/cv.pdf")
    doc.link = Link(url="http://umsu.de/papers/cv.pdf")
    doc.link.anchortext = "CV"
    doc.link.context = "CV"
    doc.content = readfile(os.path.join(testdir, "cv.txt"))
    doc.numwords = 10200
    doc.numpages = 22
    doc.meta_confidence = 92
    assert paperfilter.evaluate(doc) < 0.4
示例#5
0
def test_gooddoc_badlink():
    doc = Doc(url="http://umsu.de/papers/variations.pdf")
    doc.link = Link(url="http://umsu.de/papers/variations.pdf")
    doc.link.anchortext = "slides"
    doc.link.context = "The slides for my talk"
    doc.content = readfile(os.path.join(testdir, "attitudes.txt"))
    doc.numwords = 10200
    doc.numpages = 22
    doc.meta_confidence = 92
    assert paperfilter.evaluate(doc) < 0.8
示例#6
0
def test_gooddoc():
    doc = Doc(url="http://umsu.de/papers/variations.pdf")
    doc.link = Link(url="http://umsu.de/papers/variations.pdf")
    doc.link.anchortext = "Download"
    doc.link.context = "Foo bar"
    doc.content = readfile(os.path.join(testdir, "attitudes.txt"))
    doc.numwords = 10200
    doc.numpages = 22
    doc.meta_confidence = 92
    assert paperfilter.evaluate(doc) > 0.98
示例#7
0
def test_classify():
    mc = DocClassifier(picklefile)
    ham = Doc(url='http://umsu.de/papers/variations.pdf')
    ham.content = readfile(os.path.join(testdir, 'attitudes.txt'))
    spam = Doc(url='http://umsu.de/papers/spam.pdf')
    spam.content = """ 
       Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
       eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
       enim ad minim veniam, quis nostrud exercitation ullamco laboris
       nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor
       in reprehenderit in voluptate velit esse cillum dolore eu
       fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
       proident, sunt in culpa qui officia deserunt mollit anim id est
       laborum. 
    """
    spam.content *= 50
    mc.train([ham, spam], [True, False])
    ham.content += 'foo bar'
    prob = mc.classify(ham)
    assert prob > 0.5
示例#8
0
def test_gooddoc(setups):
    doc = Doc(url='http://umsu.de/papers/variations.pdf')
    doc.content = readfile(os.path.join(testdir, 'attitudes.txt'))
    assert philosophyfilter.evaluate(doc) > 0.6