def do_annotate(self, line):
"""annotate <template_id> <data> [-n number] [-f field]- add or test annotation (aliases: a, t)
Add a new annotation (if -f is passed) or test what would be annotated
otherwise
"""
if line.find(' ') < 0:
print(
"You must provide a valid template identifier (check output of ls_templates)"
)
print(IblTool.do_annotate.__doc__)
return
template_id, criteria = line.split(' ', 1)
t = self._load_template(template_id)
if not t:
return
criteria = self._parse_criteria(criteria)
tm = TemplateMaker(t)
selection = apply_criteria(criteria, tm)
if criteria.field:
for index in selection:
index = selection[0]
tm.annotate_fragment(index, criteria.field)
self._save_template(template_id, tm.get_template())
print("[new] (%s) %r" %
(criteria.field,
remove_annotation(tm.selected_data(index))))
else:
for n, i in enumerate(selection):
print("[%d] %r" % (n, remove_annotation(tm.selected_data(i))))
def test_annotate_multiple(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match('text to annotate'), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u'Some text to annotate here', u'Another text to annotate there']}])
def test_annotate_ignore_unpaired(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match("and that's"), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u"More text with unpaired tag <img />and that's it"]}])
def test_annotate_ignore_unpaired(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match("and that's"), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u"More text with unpaired tag <img />and that's it"]}])
def test_annotate_multiple(self):
tm = TemplateMaker(self.PAGE)
tm.annotate('field1', best_match('text to annotate'), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([(tpl, None)])
self.assertEqual(ex.extract(self.PAGE)[0],
[{u'field1': [u'Some text to annotate here', u'Another text to annotate there']}])
def do_annotate(self, line):
"""annotate <template_id> <data> [-n number] [-f field]- add or test annotation (aliases: a, t)
Add a new annotation (if -f is passed) or test what would be annotated
otherwise
"""
if line.find(' ') < 0:
print("You must provide a valid template identifier (check output of ls_templates)")
print(IblTool.do_annotate.__doc__)
return
template_id, criteria = line.split(' ', 1)
t = self._load_template(template_id)
if not t:
return
criteria = self._parse_criteria(criteria)
tm = TemplateMaker(t)
selection = apply_criteria(criteria, tm)
if criteria.field:
for index in selection:
index = selection[0]
tm.annotate_fragment(index, criteria.field)
self._save_template(template_id, tm.get_template())
print("[new] (%s) %r" % (criteria.field,
remove_annotation(tm.selected_data(index))))
else:
for n, i in enumerate(selection):
print("[%d] %r" % (n, remove_annotation(tm.selected_data(i))))
def test_annotate_multiple(self):
tm = TemplateMaker(self.PAGE)
tm.annotate("field1", best_match("text to annotate"), best_match=False)
tpl = tm.get_template()
ex = InstanceBasedLearningExtractor([tpl])
self.assertEqual(
ex.extract(self.PAGE)[0], [{u"field1": [u"Some text to annotate here", u"Another text to annotate there"]}]
)
def train_from_htmlpage(self, htmlpage, data):
assert data, "Cannot train with empty data"
tm = TemplateMaker(htmlpage)
for field, values in data.items():
if (isinstance(values, (bytes, str)) or
not hasattr(values, '__iter__')):
values = [values]
for value in values:
value = str_to_unicode(value, htmlpage.encoding)
tm.annotate(field, best_match(value))
self.add_template(tm.get_template())
def train_from_htmlpage(self, htmlpage, data):
assert data, "Cannot train with empty data"
tm = TemplateMaker(htmlpage)
for field, values in data.items():
if not hasattr(values, '__iter__'):
values = [values]
for value in values:
if isinstance(value, str):
value = value.decode(htmlpage.encoding or 'utf-8')
tm.annotate(field, best_match(value))
self.add_template(tm.get_template())
def train_from_htmlpage(self, htmlpage, data):
assert data, "Cannot train with empty data"
tm = TemplateMaker(htmlpage)
for field, values in data.items():
if (isinstance(values, (bytes, str))
or not hasattr(values, '__iter__')):
values = [values]
for value in values:
value = str_to_unicode(value, htmlpage.encoding)
tm.annotate(field, best_match(value))
self.add_template(tm.get_template())
def train(self, url=None, data=None, html=None, encoding='utf-8'):
assert data, "Cannot train with empty data"
page = self._get_page(url, encoding, html)
tm = TemplateMaker(page)
for field, values in data.items():
if not hasattr(values, '__iter__'):
values = [values]
for value in values:
if isinstance(value, str):
value = value.decode(encoding)
tm.annotate(field, best_match(value))
self.templates.append(tm.get_template())
def train(self, url=None, data=None, html=None, encoding='utf-8'):
assert data, "Cannot train with empty data"
page = self._get_page(url, encoding, html)
tm = TemplateMaker(page)
for field, values in data.items():
if not hasattr(values, '__iter__'):
values = [values]
for value in values:
if isinstance(value, str):
value = value.decode(encoding)
tm.annotate(field, best_match(value))
self.templates.append(tm.get_template())
def do_a(self, line):
"""a <template> <data> [-n number] [-f field]- add or test annotation
Add a new annotation (if -f is passed) or test what would be annotated
otherwise
"""
template_id, criteria = line.split(' ', 1)
t = self._load_template(template_id)
criteria = parse_criteria(criteria)
tm = TemplateMaker(t)
selection = apply_criteria(criteria, tm)
if criteria.field:
for index in selection:
index = selection[0]
tm.annotate_fragment(index, criteria.field)
self._save_template(template_id, tm.get_template())
print "[new] (%s) %r" % (criteria.field,
remove_annotation(tm.selected_data(index)))
else:
for n, i in enumerate(selection):
print "[%d] %r" % (n, remove_annotation(tm.selected_data(i)))
def do_a(self, line):
"""a <template> <data> [-n number] [-f field]- add or test annotation
Add a new annotation (if -f is passed) or test what would be annotated
otherwise
"""
template_id, criteria = line.split(' ', 1)
t = self._load_template(template_id)
criteria = self._parse_criteria(criteria)
tm = TemplateMaker(t)
selection = apply_criteria(criteria, tm)
if criteria.field:
for index in selection:
index = selection[0]
tm.annotate_fragment(index, criteria.field)
self._save_template(template_id, tm.get_template())
print "[new] (%s) %r" % (
criteria.field, remove_annotation(tm.selected_data(index)))
else:
for n, i in enumerate(selection):
print "[%d] %r" % (n, remove_annotation(tm.selected_data(i)))
