def test_selected_data(self): tm = TemplateMaker(self.PAGE) indexes = tm.select(best_match('text to annotate')) data = [tm.selected_data(i) for i in indexes] self.assertEqual(data, \ [u'<p>Some text to annotate here</p>', \ u'<p>Another text to annotate there</p>'])
def test_annotate_multiple(self): tm = TemplateMaker(self.PAGE) tm.annotate('field1', best_match('text to annotate'), best_match=False) tpl = tm.get_template() ex = InstanceBasedLearningExtractor([(tpl, None)]) self.assertEqual(ex.extract(self.PAGE)[0], [{u'field1': [u'Some text to annotate here', u'Another text to annotate there']}])
def test_annotations(self): tm = TemplateMaker(self.PAGE) tm.annotate('field1', best_match('text to annotate'), best_match=False) annotations = [x[0] for x in tm.annotations()] self.assertEqual(annotations, [{u'annotations': {u'content': u'field1'}}, {u'annotations': {u'content': u'field1'}}])
def test_annotate_ignore_unpaired(self): tm = TemplateMaker(self.PAGE) tm.annotate('field1', best_match("and that's"), best_match=False) tpl = tm.get_template() ex = InstanceBasedLearningExtractor([(tpl, None)]) self.assertEqual(ex.extract(self.PAGE)[0], [{u'field1': [u"More text with unpaired tag <img />and that's it"]}])
def test_annotations(self): tm = TemplateMaker(self.PAGE) tm.annotate("field1", best_match("text to annotate"), best_match=False) annotations = [x[0] for x in tm.annotations()] self.assertEqual( annotations, [{u"annotations": {u"content": u"field1"}}, {u"annotations": {u"content": u"field1"}}] )
def test_annotate_multiple(self): tm = TemplateMaker(self.PAGE) tm.annotate("field1", best_match("text to annotate"), best_match=False) tpl = tm.get_template() ex = InstanceBasedLearningExtractor([tpl]) self.assertEqual( ex.extract(self.PAGE)[0], [{u"field1": [u"Some text to annotate here", u"Another text to annotate there"]}] )
def apply_criteria(criteria, tm): """Apply the given criteria object to the given template""" func = best_match(criteria.text) if criteria.text else lambda x, y: False sel = tm.select(func) if criteria.number is not None: if criteria.number < len(sel): sel = [sel[criteria.number]] else: sel = [] return sel
def train_from_htmlpage(self, htmlpage, data): assert data, "Cannot train with empty data" tm = TemplateMaker(htmlpage) for field, values in data.items(): if (isinstance(values, (bytes, str)) or not hasattr(values, '__iter__')): values = [values] for value in values: value = str_to_unicode(value, htmlpage.encoding) tm.annotate(field, best_match(value)) self.add_template(tm.get_template())
def train_from_htmlpage(self, htmlpage, data): assert data, "Cannot train with empty data" tm = TemplateMaker(htmlpage) for field, values in data.items(): if not hasattr(values, '__iter__'): values = [values] for value in values: if isinstance(value, str): value = value.decode(htmlpage.encoding or 'utf-8') tm.annotate(field, best_match(value)) self.add_template(tm.get_template())
def train(self, url=None, data=None, html=None, encoding='utf-8'): assert data, "Cannot train with empty data" page = self._get_page(url, encoding, html) tm = TemplateMaker(page) for field, values in data.items(): if not hasattr(values, '__iter__'): values = [values] for value in values: if isinstance(value, str): value = value.decode(encoding) tm.annotate(field, best_match(value)) self.templates.append(tm.get_template())
def annotate(url, site_id, items): t = url_to_page(url) tms = [TemplateMaker(t)] for n, s in items: func = best_match(s) sel = tms[-1].select(func) print 'ATTRIBUTE: %s' % n for i in sel: print u'[%d] %s' % (i, tms[-1].selected_data(i)) if len(sel) == 1: row = sel[0] else: row = raw_input('? ') try: row = int(row) except ValueError: row = sel[0] #row = int(raw_input('? ')) #rows.pop(0) print 'SELECTED: %d' % row print '' annotated = False for tm in tms: try: if tm.annotate_fragment(row, n): annotated = True break except FragmentAlreadyAnnotated: pass if not annotated: tms.append(TemplateMaker(t)) tms[-1].annotate_fragment(row, n) save_templates('scraper.json', site_id, (tm.get_template() for tm in tms)) return [tm.get_template() for tm in tms]
def _matches(self, text): bm = best_match(text) matches = [(bm(f, self.PAGE), f) for f in self.PAGE.parsed_body] matches = [x for x in matches if x[0]] matches.sort(reverse=True) return [self.PAGE.fragment_data(x[1]) for x in matches]
def test_annotate_fragment_not_found(self): tm = TemplateMaker(self.PAGE) self.assertRaises(FragmentNotFound, tm.annotate, 'field1', best_match("missing text"))
def test_selected_data(self): tm = TemplateMaker(self.PAGE) indexes = tm.select(best_match("text to annotate")) data = [tm.selected_data(i) for i in indexes] self.assertEqual(data, [u"<p>Some text to annotate here</p>", u"<p>Another text to annotate there</p>"])
def test_annotate_fragment_already_annotated(self): tm = TemplateMaker(self.PAGE) tm.annotate("field1", best_match("text to annotate")) self.assertRaises(FragmentAlreadyAnnotated, tm.annotate, "field1", best_match("text to annotate"))
def test_annotate_fragment_not_found(self): tm = TemplateMaker(self.PAGE) self.assertRaises(FragmentNotFound, tm.annotate, "field1", best_match("missing text"))
def test_annotate_fragment_already_annotated(self): tm = TemplateMaker(self.PAGE) tm.annotate('field1', best_match('text to annotate')) self.assertRaises(FragmentAlreadyAnnotated, tm.annotate, 'field1', best_match("text to annotate"))