Example #1
0
 def execute(self):
     try:
         extracted_data = self.extract()
         output = codecs.open(self.output_file, 'w', 'utf-8')
         output.write(extracted_data)
         output.close()
     except Exception as e:
         remove_file(self.output_file)
         logger.error("Error while parsing %s -- %s" % (self.path, e))
Example #2
0
 def execute(self):
     try:
         extracted_data = self.extract()
         output = codecs.open(self.output_file, 'w', 'utf-8')
         output.write(extracted_data)
         output.close()
     except Exception as e:
         remove_file(self.output_file)
         logger.error("Error while parsing %s -- %s"%(self.path, e))
Example #3
0
def scraper_worker(scraper_task, f):
    try:
        result = resolver.make_request(scraper_task)
        processed_result = f(result)
        output = codecs.open(scraper_task.filename(), 'w', 'utf-8')
        output.write(processed_result)
        output.close()
        logger.info("Finished task %s" % (scraper_task.filename()))
    except Exception as e:
        remove_file(scraper_task.filename())
        infoline = "Scraper task  %s failed with: %s"%(scraper_task.filename(), e)
        logger.error(infoline)
    counter.value += 1
    if (counter.value % 1000) == 0:
        print("done %s...\r" % (counter.value))
Example #4
0
def scraper_worker(scraper_task, f):
    try:
        result = resolver.make_request(scraper_task)
        processed_result = f(result)
        output = codecs.open(scraper_task.filename(), 'w', 'utf-8')
        output.write(processed_result)
        output.close()
        logger.info("Finished task %s" % (scraper_task.filename()))
    except Exception as e:
        remove_file(scraper_task.filename())
        infoline = "Scraper task  %s failed with: %s" % (
            scraper_task.filename(), e)
        logger.error(infoline)
    counter.value += 1
    if (counter.value % 1000) == 0:
        print("done %s...\r" % (counter.value))