Esempio n. 1
0
 def _rule_element(self, example, element):
     try:
         pattern = self._get_element_path(example.content, element.parent)
         context = self.context_resolver.get_context(element.parent)
         pattern.insert(0, context)
         return PathRule(pattern)
     except Exception, e:
         log.warn('Path ruler cannot rule element %s: %s' #@UndefinedVariable 
                  % (str(element), e)) 
         return None
Esempio n. 2
0
 def _get_new_example_set(self, rule, example_set):
     """
     Return a list of examples with the same value attribute as example_set
     but where the content is the result of applying rule.
     """
     new_example_set = []
     for example in example_set:
         value = example.value
         content = rule.apply(example.content)
         if value and content:
             new_example_set.append(Example(value, content))
         else:
             log.warn('Example content is None after applying rule')  #@UndefinedVariable
     return new_example_set
 def _do_citeseerx(self, source, page):
     """
     Searches the page for a link to the reference, and then retrieves the
     reference.
     Returns a tuple with the full reference and its format.
     """ 
     log.info('Using CiteSeerX reference wrapper') #@UndefinedVariable
     ref = (None, None)
     
     try:
         ref_element = page.find('div', {'class':'content'},
                                 text=re.compile('@\w*{'))
         ref_element = ref_element.parent.findAll(text=True)
         reference = ''.join(ref_element)
     except Exception, e:
         log.warn('Could not find reference in citeseerx page: %s' % e) #@UndefinedVariable
         return ref
 def run(self):
     self.formatted_references = []
     for item in self.items:
         if not item.extraction.references:
             log.warn('Item has no references') #@UndefinedVariable
             continue
         
         if item.reference_entry:
             self.formatted_references.append(item.reference_entry)
             continue
     
         id = item.extraction.references[0].id
         log.debug('Formatting reference with id %d' % id) #@UndefinedVariable
         entry = self.reference_formatter.format_reference(id)
         if(entry):
             log.debug('Reference with id %d formatted' % id) #@UndefinedVariable
             self.formatted_references.append(entry)
             item.reference_entry = entry
Esempio n. 5
0
 def train(self, examples):
     """
     Generates all the possible wrappers that cover the given examples.
     
     The content of the examples must be compatible with the input type
     of the first of the rules, i.e. if the first ruler expects a string, 
     the content attribute of the examples must be a string.
     """
     wrappers = []
     if len(examples) < self.num_examples:
         log.warn('Too few examples. Could not train wrappers') #@UndefinedVariable
         return wrappers
     rule_sets = self._get_rule_sets(list(self.rulers), examples)
     for rule_set in rule_sets:
         wrapper = Wrapper(rules=rule_set)
         self._evaluate_wrapper(wrapper, examples)
         wrappers.append(wrapper)
     log.debug('Trainer generated %d wrappers (not prunned)' % #@UndefinedVariable 
               len(wrappers))
     return wrappers
Esempio n. 6
0
 def _rule_example_content(self, value, content):
     try:
         text = content.strip()
     except Exception, e:
         log.warn('Error stripping %s: %s' % (str(content)[:40], e)) #@UndefinedVariable
         return None