def execute(self, data): """Execute the crawler and create a database record of having done so.""" if Crawl.is_aborted(self.crawler, self.run_id): return try: Crawl.operation_start(self.crawler, self.stage, self.run_id) self.log.info( "[%s->%s(%s)]: %s", self.crawler.name, self.stage.name, self.stage.method_name, self.run_id, ) return self.stage.method(self, data) except QueueTooBigError as qtbe: self.emit_warning(str(qtbe)) except Exception as exc: self.emit_exception(exc) if not self.continue_on_error: raise exc finally: Crawl.operation_end(self.crawler, self.run_id) shutil.rmtree(self.work_path)
def execute(self, data): """Execute the crawler and create a database record of having done so.""" if Crawl.is_aborted(self.crawler, self.run_id): return try: Crawl.operation_start(self.crawler, self.stage, self.run_id) self.log.info('[%s->%s(%s)]: %s', self.crawler.name, self.stage.name, self.stage.method_name, self.run_id) return self.stage.method(self, data) except Exception as exc: self.emit_exception(exc) finally: Crawl.operation_end(self.crawler, self.run_id) shutil.rmtree(self.work_path)