def save(cls, crawler, stage, level, run_id, error=None, message=None):
"""Create an event, possibly based on an exception."""
event = {
'stage': stage.name,
'level': level,
'timestamp': pack_now(),
'error': error,
'message': message
}
data = dump_json(event)
keys = [
make_key(crawler, "events"),
make_key(crawler, "events", level),
make_key(crawler, "events", stage),
make_key(crawler, "events", stage, level),
make_key(crawler, "events", run_id),
make_key(crawler, "events", run_id, level),
]
for key in keys:
conn.lpush(key, data)
conn.expire(key, REDIS_EXPIRE)
# Persist the counts for longer
count_keys = [
make_key(crawler, "events", "count", level),
make_key(crawler, "events", "count", stage, level),
make_key(crawler, "events", "count", run_id, level),
]
for key in count_keys:
conn.incr(key)
conn.expire(key, REDIS_LONG)
return event
def operation_start(cls, crawler, stage, run_id):
if not conn.sismember(make_key(crawler, "runs"), run_id):
conn.sadd(make_key(crawler, "runs"), run_id)
conn.set(make_key("run", run_id, "start"), pack_now())
conn.incr(make_key("run", run_id))
conn.incr(make_key("run", run_id, "total_ops"))
conn.incr(make_key(crawler, stage))
conn.incr(make_key(crawler, "total_ops"))
conn.set(make_key(crawler, "last_run"), pack_now())
conn.set(make_key(crawler, "current_run"), run_id)
def queue(cls, stage, state, data, delay=None):
crawler = state.get('crawler')
task_data = cls.serialize_task_data(stage, state, data, delay)
conn.rpush(make_key('queue', crawler, stage), task_data)
conn.expire(make_key('queue', crawler, stage), QUEUE_EXPIRE)
conn.incr(make_key('queue_pending', crawler))