def test_with_tokenization(self): t = PrefTree() t.add_url(prepare_url('eprint.iacr.org/2016/093'), False) t.add_url(prepare_url('eprint.iacr.org/2016/093.pdf'), True) t.add_url(prepare_url('eprint.iacr.org/2015/1248.pdf'), True) t.add_url(prepare_url('eprint.iacr.org/2015/1248'), False) t.print_as_tree() self.assertEqual(t.match(prepare_url('eprint.iacr.org/2014/528.pdf')), (2,2)) self.assertEqual(t.match(prepare_url('eprint.iacr.org/2014/528')), (2,0))
def test_wildcard(self): t = PrefTree() t.add_url('arxiv.org/pdf/', True) t['arxiv.org/pdf/'].is_wildcard = True self.assertTrue(t.check_sanity()) self.assertTrue(t.has_wildcard()) self.assertEqual(t.match('arxiv.org/pdf/1410.1454v2'), (1,1)) t.add_url('arxiv.org/pdf/1412.8548v1', True) self.assertEqual(t.match('arxiv.org/pdf/1410.1454v2'), (2,2)) t.print_as_tree() self.assertEqual(len(t.urls()), 1)
def test_create(self): t = PrefTree() urls = ['aaba','cadb','abdc','abcd','afgh','abec'] for u in urls: t.add_url(u) self.assertTrue(t.check_sanity()) self.assertFalse(t.has_wildcard()) t.print_as_tree() self.assertEqual(sorted([flatten(u) for u, c, s in t.urls()]), sorted(urls)) for u in urls: self.assertEqual(t.match(u), (1,0)) self.assertEqual(t.match('bac'), (0,0))
def test_prune_with_reverse(self): t = PrefTree() for url, success in [ ('researchgate.net/publication/233865122_uriset', False), ('researchgate.net/publication/143874230_albtedru', False), ('researchgate.net/publication/320748374_kelbcad', False), ('researchgate.net/publication/233865122_uriset.pdf', True), ('researchgate.net/publication/143874230_albtedru.pdf', True), ('researchgate.net/publication/320748374_kelbcad.pdf', True), ('onlinelibrary.wiley.com/wol1/doi/10.1002/anie.200800037.abstract', False), ('onlinelibrary.wiley.com/wol1/doi/10.1002/anie.200800037.pdf', False)]: t.add_url(url, success) t.print_as_tree() t, pruned = t.prune(reverse=True) t.print_as_tree() self.assertTrue(t.check_sanity()) for u, c, s in t.urls(): print flatten(u), c, s
def test_prune(self): t = PrefTree() with self.assertRaises(ValueError): t.prune(min_urls=0) for url, success in [ ('arxiv.org/pdf/1410.1234', True), ('arxiv.org/pdf/1409.1094', True), ('arxiv.org/pdf/1201.5480', True), ('arxiv.org/pdf/1601.01234', True), ('arxiv.org/pdf/1602.01i34', False), # oops ]: t.add_url(url, success) t, pruned = t.prune(min_rate=0.75,min_children=2,min_urls=1) self.assertEqual(len(t.urls()), 1) self.assertTrue(t.has_wildcard()) self.assertEqual(t.match('arxiv.org/pdf/1784.1920'), (5,4)) self.assertEqual(t.match('arxiv.org/pdf/2340.0124'), (0,0)) self.assertTrue(t.predict_success('arxiv.org/pdf/1784.1920', threshold=0.6, min_urls=3)) t.print_as_tree()