def get(self, section): """Get option. @param section: section to fetch. @return: option value. """ try: return getattr(self, section) except AttributeError as e: logger.error("Option %s is not found in configuration, error: %s" % (section, e))
def run(self): list_turples = list() try: if self.checkConnection(): data = requests.get(self.paste_url) parser = HTMLParser(encoding='utf8') tree = parse(StringIO(data.content), parser) root = tree.getroot() links = root.xpath(self.links_xpath) if self.links_post_process is not None: links = map(self.links_post_process, links) for l in links: que_url = self.query_url % l worker_turple = (self.grabber_name, que_url) list_turples.append(worker_turple) else: return None except Exception, e: msg = "%s: Exception running scraper [%s]" % ( self.grabber_name, str(e) ) logger.error(msg) return None