def runs(cls, crawler):
runs = []
for run_id in cls.run_ids(crawler):
start = conn.get(make_key(crawler, "run", run_id, "start"))
end = conn.get(make_key(crawler, "run", run_id, "end"))
total_ops = conn.get(make_key(crawler, "run", run_id, "total_ops"))
runs.append({
"run_id": run_id,
"total_ops": unpack_int(total_ops),
"start": unpack_datetime(start, datetime.utcnow()),
"end": unpack_datetime(end),
})
return runs
def test_datetime(self):
date = datetime.utcnow().date()
packed = pack_datetime(date)
unpacked = unpack_datetime(packed)
assert unpacked == date
now = pack_now()
assert now.startswith(packed), now
time = datetime.utcnow()
packed = pack_datetime(time)
unpacked = unpack_datetime(packed)
assert unpacked == time
assert pack_datetime("Banana") is None
assert unpack_datetime("Banana") is None
def event_list(cls, key, start, end):
results = []
events = conn.lrange(key, start, end)
if events is None:
return results
for event in events:
result = load_json(event)
result["timestamp"] = unpack_datetime(result['timestamp'])
results.append(result)
return results
def last_run(cls, crawler):
last_run = conn.get(make_key(crawler, "last_run"))
return unpack_datetime(last_run)