def execute(self, data):
"""Execute the crawler and create a database record of having done
so."""
if Crawl.is_aborted(self.crawler, self.run_id):
return
try:
Crawl.operation_start(self.crawler, self.stage, self.run_id)
self.log.info(
"[%s->%s(%s)]: %s",
self.crawler.name,
self.stage.name,
self.stage.method_name,
self.run_id,
)
return self.stage.method(self, data)
except QueueTooBigError as qtbe:
self.emit_warning(str(qtbe))
except Exception as exc:
self.emit_exception(exc)
if not self.continue_on_error:
raise exc
finally:
Crawl.operation_end(self.crawler, self.run_id)
shutil.rmtree(self.work_path)
def test_operation_reporting(self, crawler, context):
stage = list(crawler.stages)[0]
Crawl.operation_start(crawler, stage, context.run_id)
assert crawler.latest_runid == context.run_id
assert len(list(crawler.runs)) == 1
assert crawler.op_count == 1
assert isinstance(crawler.last_run, datetime.datetime)
Crawl.operation_end(crawler, context.run_id)
crawler.flush()
assert crawler.op_count == 0
assert len(list(crawler.runs)) == 0
def execute(self, data):
"""Execute the crawler and create a database record of having done
so."""
if Crawl.is_aborted(self.crawler, self.run_id):
return
try:
Crawl.operation_start(self.crawler, self.stage, self.run_id)
self.log.info('[%s->%s(%s)]: %s', self.crawler.name,
self.stage.name, self.stage.method_name, self.run_id)
return self.stage.method(self, data)
except Exception as exc:
self.emit_exception(exc)
finally:
Crawl.operation_end(self.crawler, self.run_id)
shutil.rmtree(self.work_path)
