def op_count(cls, crawler, stage=None):
"""Total operations performed for this crawler"""
if stage:
total_ops = cls.conn.get(make_key(crawler, stage))
else:
total_ops = cls.conn.get(make_key(crawler, "total_ops"))
return unpack_int(total_ops)
def runs(cls, crawler):
for run_id in cls.run_ids(crawler):
start = cls.conn.get(make_key("run", run_id, "start"))
end = cls.conn.get(make_key("run", run_id, "end"))
total_ops = cls.conn.get(make_key("run", run_id, "total_ops"))
yield {
'run_id': run_id,
'total_ops': unpack_int(total_ops),
'start': unpack_datetime(start, datetime.utcnow()),
'end': unpack_datetime(end)
}
def runs(cls, crawler):
for run_id in cls.conn.lrange(make_key(crawler, "runs_list"), 0, -1):
start = cls.conn.get(make_key("run", run_id, "start"))
end = cls.conn.get(make_key("run", run_id, "end"))
total_ops = cls.conn.get(make_key("run", run_id, "total_ops"))
yield {
'run_id': run_id,
'total_ops': unpack_int(total_ops),
'start': unpack_datetime(start),
'end': unpack_datetime(end)
}
def size(cls, crawler):
"""Total operations pending for this crawler"""
key = make_key('queue_pending', crawler)
return unpack_int(cls.conn.get(key))
def operation_end(cls, crawler, run_id):
cls.conn.set(make_key(crawler, "last_run"), pack_now())
pending = cls.conn.decr(make_key("run", run_id))
if unpack_int(pending) == 0:
cls.conn.set(make_key("run", run_id, "end"), pack_now())