def get_wrappers(self, url=u'%', field=u'%'):
"""
Gets all the wrappers from a given collection.
"""
url = unicode(url)
field = unicode(field)
collection = self.find_wrapper_collection(url, field)
if not collection:
return []
rule_factory = RuleFactory()
wrappers = []
for m_wrapper in collection.wrappers:
wrapper = Wrapper()
wrapper.id = m_wrapper.id
wrapper.upvotes = m_wrapper.upvotes
wrapper.downvotes = m_wrapper.downvotes
for rule in m_wrapper.rules:
pattern = simplejson.loads(str(rule.pattern))
r_rule = rule_factory.create_rule(rule.rule_type, pattern)
if r_rule:
wrapper.add_rule(r_rule)
wrappers.append(wrapper)
return wrappers
def test_persist_and_update_wrapper(self):
wrapper = Wrapper()
wrapper.add_rule(MockRule01(33))
wrapper.add_rule(MockRule01(55))
wrapper.add_rule(MockRule02([1, [2, 3, 4, 5], 6]))
self.wm.persist_wrapper(u'concrete_url', u'concrete_field', wrapper)
# Get wrappers
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
# Update wrapper
wrapper = wrappers[0]
wrapper.upvotes += 1
wrapper.downvotes -= 1
self.wm.update_wrapper(wrapper)
# Get the wrapper again
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
self.failUnless(wrappers[0].upvotes == 1)
self.failUnless(wrappers[0].downvotes == -1)
# Update wrapper rules
wrapper = wrappers[0]
wrapper.rules[0].pattern = 223
wrapper.rules[2].pattern = [1, 6]
self.wm.update_wrapper(wrapper)
# Get the wrapper again
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
self.failUnless(wrappers[0].rules[0].pattern == 223)
self.failUnless(wrappers[0].rules[2].pattern == [1, 6])
# Add another wrapper rule
wrapper = wrappers[0]
wrapper.rules.append(MockRule02([2, 3, 4, 5]))
self.wm.update_wrapper(wrapper)
# Get the wrapper again
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
self.failUnless(len(wrappers[0].rules) == 4)
self.failUnless(wrappers[0].rules[3].pattern == [2, 3, 4, 5])
def test_evaluate_wrapper(self):
self.example_set = [
Example('c01_r22_r33', 'c01'),
Example('c02_r22_r33', 'c02'),
Example('incorrect_value', 'c03')
]
wrapper = Wrapper(rules=[MockRule(22), MockRule(33)])
self.wt._evaluate_wrapper(wrapper, self.example_set)
self.failUnless(wrapper.upvotes == 2)
self.failUnless(wrapper.downvotes == 1)
def train(self, examples):
"""
Generates all the possible wrappers that cover the given examples.
The content of the examples must be compatible with the input type
of the first of the rules, i.e. if the first ruler expects a string,
the content attribute of the examples must be a string.
"""
wrappers = []
if len(examples) < self.num_examples:
log.warn('Too few examples. Could not train wrappers') #@UndefinedVariable
return wrappers
rule_sets = self._get_rule_sets(list(self.rulers), examples)
for rule_set in rule_sets:
wrapper = Wrapper(rules=rule_set)
self._evaluate_wrapper(wrapper, examples)
wrappers.append(wrapper)
log.debug('Trainer generated %d wrappers (not prunned)' % #@UndefinedVariable
len(wrappers))
return wrappers
def test_persist_and_update_wrapper(self):
wrapper = Wrapper()
wrapper.add_rule(MockRule01(33))
wrapper.add_rule(MockRule01(55))
wrapper.add_rule(MockRule02([1, [2, 3, 4, 5], 6]))
self.wm.persist_wrapper(u'concrete_url', u'concrete_field', wrapper)
# Get wrappers
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
# Update wrapper
wrapper = wrappers[0]
wrapper.upvotes += 1
wrapper.downvotes -= 1
self.wm.update_wrapper(wrapper)
# Get the wrapper again
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
self.failUnless(wrappers[0].upvotes == 1)
self.failUnless(wrappers[0].downvotes == -1)
# Update wrapper rules
wrapper = wrappers[0]
wrapper.rules[0].pattern = 223
wrapper.rules[2].pattern = [1, 6]
self.wm.update_wrapper(wrapper)
# Get the wrapper again
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
self.failUnless(wrappers[0].rules[0].pattern == 223)
self.failUnless(wrappers[0].rules[2].pattern == [1, 6])
# Add another wrapper rule
wrapper = wrappers[0]
wrapper.rules.append(MockRule02([2, 3, 4, 5]))
self.wm.update_wrapper(wrapper)
# Get the wrapper again
wrappers = self.wm.get_wrappers(u'concrete_url', u'concrete_field')
self.failUnless(len(wrappers) == 1)
self.failUnless(len(wrappers[0].rules) == 4)
self.failUnless(wrappers[0].rules[3].pattern == [2, 3, 4, 5])
def xtest_persist_and_get_wrapper(self):
wrapper = Wrapper()
wrapper.add_rule(MockRule01(33))
wrapper.add_rule(MockRule01(55))
wrapper.add_rule(MockRule02([1, [2, 3, 4, 5], 6]))
self.wm.persist_wrapper(u'some_url', u'some_field', wrapper)
wrapper = Wrapper()
wrapper.add_rule(MockRule01(66))
wrapper.add_rule(MockRule01(77))
wrapper.add_rule(MockRule02([[2, 3, 4, 5], 4]))
self.wm.persist_wrapper(u'some_url', u'some_field', wrapper)
wrapper = Wrapper()
wrapper.add_rule(MockRule01(11))
wrapper.add_rule(MockRule01(22))
wrapper.add_rule(MockRule01(33))
self.wm.persist_wrapper(u'some_url', u'some_other_field', wrapper)
# Get non-existent wrapper
wrappers = self.wm.get_wrappers(u'some_url', u'non_existent_field')
self.failIf(wrappers, 'Get non-existent wrapper')
# Get wrappers
wrappers = self.wm.get_wrappers(u'some_url', u'some_field')
self.failUnless(len(wrappers) == 2)
wrappers = self.wm.get_wrappers(u'some_url', u'some_other_field')
self.failUnless(len(wrappers) == 1)
def xtest_persist_wrapper_with_incorrect_rules(self):
wrapper = Wrapper()
wrapper.add_rule(MockRule01(MockRule02(33)))
self.failUnlessRaises(TypeError, self.wm.persist_wrapper,
u'some_url', u'some_field', wrapper)
def xtest_persist_and_get_wrapper(self):
wrapper = Wrapper()
wrapper.add_rule(MockRule01(33))
wrapper.add_rule(MockRule01(55))
wrapper.add_rule(MockRule02([1, [2, 3, 4, 5], 6]))
self.wm.persist_wrapper(u'some_url', u'some_field', wrapper)
wrapper = Wrapper()
wrapper.add_rule(MockRule01(66))
wrapper.add_rule(MockRule01(77))
wrapper.add_rule(MockRule02([[2, 3, 4, 5], 4]))
self.wm.persist_wrapper(u'some_url', u'some_field', wrapper)
wrapper = Wrapper()
wrapper.add_rule(MockRule01(11))
wrapper.add_rule(MockRule01(22))
wrapper.add_rule(MockRule01(33))
self.wm.persist_wrapper(u'some_url', u'some_other_field', wrapper)
# Get non-existent wrapper
wrappers = self.wm.get_wrappers(u'some_url', u'non_existent_field')
self.failIf(wrappers, 'Get non-existent wrapper')
# Get wrappers
wrappers = self.wm.get_wrappers(u'some_url', u'some_field')
self.failUnless(len(wrappers) == 2)
wrappers = self.wm.get_wrappers(u'some_url', u'some_other_field')
self.failUnless(len(wrappers) == 1)
def xtest_persist_wrapper_with_incorrect_rules(self):
wrapper = Wrapper()
wrapper.add_rule(MockRule01(MockRule02(33)))
self.failUnlessRaises(TypeError, self.wm.persist_wrapper, u'some_url',
u'some_field', wrapper)
