def test_selected_data(self):
tm = TemplateMaker(self.PAGE)
indexes = tm.select(best_match('text to annotate'))
data = [tm.selected_data(i) for i in indexes]
self.assertEqual(data, \
[u'<p>Some text to annotate here</p>', \
u'<p>Another text to annotate there</p>'])
def test_annotate_multiple(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match('text to annotate'), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u'Some text to annotate here', u'Another text to annotate there']}])
def test_annotations(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match('text to annotate'), best_match=False)
annotations = [x[0] for x in tm.annotations()]
self.assertEqual(annotations,
[{u'annotations': {u'content': u'field1'}},
{u'annotations': {u'content': u'field1'}}])
def test_annotations(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match('text to annotate'), best_match=False)
annotations = [x[0] for x in tm.annotations()]
self.assertEqual(annotations,
[{u'annotations': {u'content': u'field1'}},
{u'annotations': {u'content': u'field1'}}])
def test_selected_data(self):
tm = TemplateMaker(self.PAGE)
indexes = tm.select(best_match('text to annotate'))
data = [tm.selected_data(i) for i in indexes]
self.assertEqual(data, \
[u'<p>Some text to annotate here</p>', \
u'<p>Another text to annotate there</p>'])
def test_annotate_ignore_unpaired(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match("and that's"), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u"More text with unpaired tag <img />and that's it"]}])
def test_annotations(self):
tm = TemplateMaker(self.PAGE)
tm.annotate("field1", best_match("text to annotate"), best_match=False)
annotations = [x[0] for x in tm.annotations()]
self.assertEqual(
annotations, [{u"annotations": {u"content": u"field1"}}, {u"annotations": {u"content": u"field1"}}]
)
def test_annotate_ignore_unpaired(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match("and that's"), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u"More text with unpaired tag <img />and that's it"]}])
def test_annotate_multiple(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match('text to annotate'), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u'Some text to annotate here', u'Another text to annotate there']}])
def test_annotate_multiple(self):
tm = TemplateMaker(self.PAGE)
tm.annotate("field1", best_match("text to annotate"), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([tpl])
self.assertEqual(
ex.extract(self.PAGE)[0], [{u"field1": [u"Some text to annotate here", u"Another text to annotate there"]}]
)
def apply_criteria(criteria, tm):
"""Apply the given criteria object to the given template"""
func = best_match(criteria.text) if criteria.text else lambda x, y: False
sel = tm.select(func)
if criteria.number is not None:
if criteria.number < len(sel):
sel = [sel[criteria.number]]
else:
sel = []
return sel
def apply_criteria(criteria, tm):
"""Apply the given criteria object to the given template"""
func = best_match(criteria.text) if criteria.text else lambda x, y: False
sel = tm.select(func)
if criteria.number is not None:
if criteria.number < len(sel):
sel = [sel[criteria.number]]
else:
sel = []
return sel
def train_from_htmlpage(self, htmlpage, data):
assert data, "Cannot train with empty data"
tm = TemplateMaker(htmlpage)
for field, values in data.items():
if (isinstance(values, (bytes, str)) or
not hasattr(values, '__iter__')):
values = [values]
for value in values:
value = str_to_unicode(value, htmlpage.encoding)
tm.annotate(field, best_match(value))
self.add_template(tm.get_template())
def train_from_htmlpage(self, htmlpage, data):
assert data, "Cannot train with empty data"
tm = TemplateMaker(htmlpage)
for field, values in data.items():
if not hasattr(values, '__iter__'):
values = [values]
for value in values:
if isinstance(value, str):
value = value.decode(htmlpage.encoding or 'utf-8')
tm.annotate(field, best_match(value))
self.add_template(tm.get_template())
def train_from_htmlpage(self, htmlpage, data):
assert data, "Cannot train with empty data"
tm = TemplateMaker(htmlpage)
for field, values in data.items():
if (isinstance(values, (bytes, str))
or not hasattr(values, '__iter__')):
values = [values]
for value in values:
value = str_to_unicode(value, htmlpage.encoding)
tm.annotate(field, best_match(value))
self.add_template(tm.get_template())
def train(self, url=None, data=None, html=None, encoding='utf-8'):
assert data, "Cannot train with empty data"
page = self._get_page(url, encoding, html)
tm = TemplateMaker(page)
for field, values in data.items():
if not hasattr(values, '__iter__'):
values = [values]
for value in values:
if isinstance(value, str):
value = value.decode(encoding)
tm.annotate(field, best_match(value))
self.templates.append(tm.get_template())
def train(self, url=None, data=None, html=None, encoding='utf-8'):
assert data, "Cannot train with empty data"
page = self._get_page(url, encoding, html)
tm = TemplateMaker(page)
for field, values in data.items():
if not hasattr(values, '__iter__'):
values = [values]
for value in values:
if isinstance(value, str):
value = value.decode(encoding)
tm.annotate(field, best_match(value))
self.templates.append(tm.get_template())
def annotate(url, site_id, items):
t = url_to_page(url)
tms = [TemplateMaker(t)]
for n, s in items:
func = best_match(s)
sel = tms[-1].select(func)
print 'ATTRIBUTE: %s' % n
for i in sel:
print u'[%d] %s' % (i, tms[-1].selected_data(i))
if len(sel) == 1:
row = sel[0]
else:
row = raw_input('? ')
try:
row = int(row)
except ValueError:
row = sel[0]
#row = int(raw_input('? ')) #rows.pop(0)
print 'SELECTED: %d' % row
print ''
annotated = False
for tm in tms:
try:
if tm.annotate_fragment(row, n):
annotated = True
break
except FragmentAlreadyAnnotated:
pass
if not annotated:
tms.append(TemplateMaker(t))
tms[-1].annotate_fragment(row, n)
save_templates('scraper.json', site_id, (tm.get_template() for tm in tms))
return [tm.get_template() for tm in tms]
def annotate(url, site_id, items):
t = url_to_page(url)
tms = [TemplateMaker(t)]
for n, s in items:
func = best_match(s)
sel = tms[-1].select(func)
print 'ATTRIBUTE: %s' % n
for i in sel:
print u'[%d] %s' % (i, tms[-1].selected_data(i))
if len(sel) == 1:
row = sel[0]
else:
row = raw_input('? ')
try:
row = int(row)
except ValueError:
row = sel[0]
#row = int(raw_input('? ')) #rows.pop(0)
print 'SELECTED: %d' % row
print ''
annotated = False
for tm in tms:
try:
if tm.annotate_fragment(row, n):
annotated = True
break
except FragmentAlreadyAnnotated:
pass
if not annotated:
tms.append(TemplateMaker(t))
tms[-1].annotate_fragment(row, n)
save_templates('scraper.json', site_id, (tm.get_template() for tm in tms))
return [tm.get_template() for tm in tms]
def _matches(self, text):
bm = best_match(text)
matches = [(bm(f, self.PAGE), f) for f in self.PAGE.parsed_body]
matches = [x for x in matches if x[0]]
matches.sort(reverse=True)
return [self.PAGE.fragment_data(x[1]) for x in matches]
def test_annotate_fragment_not_found(self):
tm = TemplateMaker(self.PAGE)
self.assertRaises(FragmentNotFound, tm.annotate, 'field1', best_match("missing text"))
def test_selected_data(self):
tm = TemplateMaker(self.PAGE)
indexes = tm.select(best_match("text to annotate"))
data = [tm.selected_data(i) for i in indexes]
self.assertEqual(data, [u"<p>Some text to annotate here</p>", u"<p>Another text to annotate there</p>"])
def test_annotate_fragment_already_annotated(self):
tm = TemplateMaker(self.PAGE)
tm.annotate("field1", best_match("text to annotate"))
self.assertRaises(FragmentAlreadyAnnotated, tm.annotate, "field1", best_match("text to annotate"))
def test_annotate_fragment_not_found(self):
tm = TemplateMaker(self.PAGE)
self.assertRaises(FragmentNotFound, tm.annotate, "field1", best_match("missing text"))
def test_annotate_fragment_already_annotated(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match('text to annotate'))
self.assertRaises(FragmentAlreadyAnnotated, tm.annotate, 'field1', best_match("text to annotate"))
def _matches(self, text):
bm = best_match(text)
matches = [(bm(f, self.PAGE), f) for f in self.PAGE.parsed_body]
matches = [x for x in matches if x[0]]
matches.sort(reverse=True)
return [self.PAGE.fragment_data(x[1]) for x in matches]
