def execute(self):
try:
extracted_data = self.extract()
output = codecs.open(self.output_file, 'w', 'utf-8')
output.write(extracted_data)
output.close()
except Exception as e:
remove_file(self.output_file)
logger.error("Error while parsing %s -- %s" % (self.path, e))
def execute(self):
try:
extracted_data = self.extract()
output = codecs.open(self.output_file, 'w', 'utf-8')
output.write(extracted_data)
output.close()
except Exception as e:
remove_file(self.output_file)
logger.error("Error while parsing %s -- %s"%(self.path, e))
def scraper_worker(scraper_task, f):
try:
result = resolver.make_request(scraper_task)
processed_result = f(result)
output = codecs.open(scraper_task.filename(), 'w', 'utf-8')
output.write(processed_result)
output.close()
logger.info("Finished task %s" % (scraper_task.filename()))
except Exception as e:
remove_file(scraper_task.filename())
infoline = "Scraper task %s failed with: %s"%(scraper_task.filename(), e)
logger.error(infoline)
counter.value += 1
if (counter.value % 1000) == 0:
print("done %s...\r" % (counter.value))
def scraper_worker(scraper_task, f):
try:
result = resolver.make_request(scraper_task)
processed_result = f(result)
output = codecs.open(scraper_task.filename(), 'w', 'utf-8')
output.write(processed_result)
output.close()
logger.info("Finished task %s" % (scraper_task.filename()))
except Exception as e:
remove_file(scraper_task.filename())
infoline = "Scraper task %s failed with: %s" % (
scraper_task.filename(), e)
logger.error(infoline)
counter.value += 1
if (counter.value % 1000) == 0:
print("done %s...\r" % (counter.value))
