def get(self, section):
"""Get option.
@param section: section to fetch.
@return: option value.
"""
try:
return getattr(self, section)
except AttributeError as e:
logger.error("Option %s is not found in configuration, error: %s" % (section, e))
def run(self):
list_turples = list()
try:
if self.checkConnection():
data = requests.get(self.paste_url)
parser = HTMLParser(encoding='utf8')
tree = parse(StringIO(data.content), parser)
root = tree.getroot()
links = root.xpath(self.links_xpath)
if self.links_post_process is not None:
links = map(self.links_post_process, links)
for l in links:
que_url = self.query_url % l
worker_turple = (self.grabber_name, que_url)
list_turples.append(worker_turple)
else:
return None
except Exception, e:
msg = "%s: Exception running scraper [%s]" % (
self.grabber_name, str(e)
)
logger.error(msg)
return None