Пример #1
0
 def get(self, section):
     """Get option.
     @param section: section to fetch.
     @return: option value.
     """
     try:
         return getattr(self, section)
     except AttributeError as e:
         logger.error("Option %s is not found in configuration, error: %s" % (section, e))
Пример #2
0
    def run(self):
            list_turples = list()
            try:
                if self.checkConnection():
                    data = requests.get(self.paste_url)
                    parser = HTMLParser(encoding='utf8')
                    tree = parse(StringIO(data.content), parser)
                    root = tree.getroot()
                    links = root.xpath(self.links_xpath)
                    if self.links_post_process is not None:
                        links = map(self.links_post_process, links)
                    for l in links:
                        que_url = self.query_url % l
                        worker_turple = (self.grabber_name, que_url)
                        list_turples.append(worker_turple)
                else:
                    return None

            except Exception, e:
                msg = "%s: Exception running scraper [%s]" % (
                    self.grabber_name, str(e)
                    )
                logger.error(msg)
                return None