def operation_start(cls, crawler, stage, run_id):
if not cls.conn.sismember(make_key(crawler, "runs"), run_id):
cls.conn.sadd(make_key(crawler, "runs"), run_id)
cls.conn.set(make_key("run", run_id, "start"), pack_now())
cls.conn.incr(make_key("run", run_id))
cls.conn.incr(make_key("run", run_id, "total_ops"))
cls.conn.incr(make_key(crawler, stage))
cls.conn.incr(make_key(crawler, "total_ops"))
cls.conn.set(make_key(crawler, "last_run"), pack_now())
cls.conn.set(make_key(crawler, "current_run"), run_id)
def record_operation_start(cls, crawler, run_id):
if not cls.conn.sismember(make_key(crawler, "runs"), run_id):
cls.conn.sadd(make_key(crawler, "runs"), run_id)
cls.conn.lpush(make_key(crawler, "runs_list"), run_id)
cls.conn.set(make_key("run", run_id, "start"), pack_now())
cls.conn.incr(make_key("run", run_id))
cls.conn.incr(make_key("run", run_id, "total_ops"))
def save(cls, crawler, stage, level, run_id, error=None, message=None):
"""Create an event, possibly based on an exception."""
event = {
'stage': stage.name,
'level': level,
'timestamp': pack_now(),
'error': error,
'message': message
}
data = dump_json(event)
cls.conn.lpush(make_key(crawler, "events"), data)
cls.conn.lpush(make_key(crawler, "events", level), data)
cls.conn.lpush(make_key(crawler, "events", stage), data)
cls.conn.lpush(make_key(crawler, "events", stage, level), data)
cls.conn.lpush(make_key(crawler, "events", run_id), data)
cls.conn.lpush(make_key(crawler, "events", run_id, level), data)
return event
def abort_run(cls, crawler, run_id):
cls.conn.sadd(make_key(crawler, "runs_abort"), run_id)
if cls.conn.get(make_key("run", run_id, "end")) is None:
cls.conn.set(make_key("run", run_id, "end"), pack_now())
def operation_end(cls, crawler, run_id):
cls.conn.set(make_key(crawler, "last_run"), pack_now())
pending = cls.conn.decr(make_key("run", run_id))
if unpack_int(pending) == 0:
cls.conn.set(make_key("run", run_id, "end"), pack_now())
def record_operation_end(cls, crawler):
cls.conn.set(make_key(crawler, "last_run"), pack_now())
def record_operation_start(cls, crawler, stage):
cls.conn.incr(make_key(crawler, stage))
cls.conn.incr(make_key(crawler, "total_ops"))
cls.conn.set(make_key(crawler, "last_run"), pack_now())