def func_wrapper(context, data, *a, **kw):
op = Operation()
op.crawler = context.crawler.name
op.name = context.stage.name
op.run_id = context.run_id
op.status = Operation.STATUS_PENDING
session.add(op)
session.commit()
context.operation_id = op.id
try:
context.log.info('Running: %s', op.name)
res = func(context, data, *a, **kw)
op.status = Operation.STATUS_SUCCESS
return res
except Exception as exc:
# this should clear results and tags created by this op
session.rollback()
Event.save(op.id, Event.LEVEL_ERROR, exc=exc)
context.log.exception(exc)
finally:
if op.status == Operation.STATUS_PENDING:
op.status = Operation.STATUS_FAILED
op.ended_at = datetime.utcnow()
session.add(op)
session.commit()
def execute(self, data):
"""Execute the crawler and create a database record of having done
so."""
op = Operation()
op.crawler = self.crawler.name
op.name = self.stage.name
op.run_id = self.run_id
op.status = Operation.STATUS_PENDING
session.add(op)
session.commit()
self.operation_id = op.id
try:
self.log.debug('Running: %s', op.name)
res = self.stage.method(self, data)
op.status = Operation.STATUS_SUCCESS
return res
except Exception as exc:
# this should clear results and tags created by this op
# TODO: should we also use transactions on the datastore?
session.rollback()
self.emit_exception(exc)
finally:
if op.status == Operation.STATUS_PENDING:
op.status = Operation.STATUS_FAILED
op.ended_at = datetime.utcnow()
session.add(op)
session.commit()
def save(cls, crawler, prev_stage, next_stage, data):
obj = cls()
obj.crawler = crawler.name
obj.prev_stage = prev_stage
obj.next_stage = next_stage
obj.data = data
session.add(obj)
return obj
def save(cls, crawler, key, value):
obj = cls.find(crawler, key)
if obj is None:
obj = cls()
obj.crawler = crawler.name
obj.key = key
obj.value = value
obj.timestamp = datetime.utcnow()
session.add(obj)
return obj
def save(cls, crawler, operation_id, prev_stage, next_stage, data):
obj = cls()
obj.crawler = crawler.name
obj.operation_id = operation_id
obj.prev_stage = prev_stage
obj.next_stage = next_stage
obj.data = data
session.add(obj)
session.flush()
return obj
def save(cls,
crawler,
operation_id,
level,
error_type=None,
error_message=None,
error_details=None):
"""Create an event, possibly based on an exception."""
event = cls()
event.crawler = crawler
event.operation_id = operation_id
event.level = level
event.error_type = error_type
event.error_message = error_message
event.error_details = error_details
session.add(event)
session.flush()
return event