def get_status(self):
"""Get the current status."""
keys = (self.pending_key, self.running_key, self.finished_key)
pending, running, finished = self.conn.mget(keys)
return {
'job_id': self.job.id,
'stage': self.stage,
'pending': max(0, unpack_int(pending)),
'running': max(0, unpack_int(running)),
'finished': max(0, unpack_int(finished)),
}
def get_status(self):
"""Get the current status."""
keys = (self.pending_key, self.running_key, self.finished_key)
pending, running, finished = self.conn.mget(keys)
return {
"job_id": self.job.id,
"stage": self.stage,
"pending": max(0, unpack_int(pending)),
"running": max(0, unpack_int(running)),
"finished": max(0, unpack_int(finished)),
}
def operation_end(cls, crawler, run_id):
conn.set(make_key(crawler, "last_run"), pack_now(), ex=REDIS_LONG)
pending = conn.decr(make_key(crawler, "run", run_id))
if unpack_int(pending) == 0:
conn.set(make_key(crawler, "run", run_id, "end"),
pack_now(),
ex=REDIS_LONG)
def op_count(cls, crawler, stage=None):
"""Total operations performed for this crawler"""
if stage:
total_ops = conn.get(make_key(crawler, stage))
else:
total_ops = conn.get(make_key(crawler, "total_ops"))
return unpack_int(total_ops)
def runs(cls, crawler):
runs = []
for run_id in cls.run_ids(crawler):
start = conn.get(make_key(crawler, "run", run_id, "start"))
end = conn.get(make_key(crawler, "run", run_id, "end"))
total_ops = conn.get(make_key(crawler, "run", run_id, "total_ops"))
runs.append({
"run_id": run_id,
"total_ops": unpack_int(total_ops),
"start": unpack_datetime(start, datetime.utcnow()),
"end": unpack_datetime(end),
})
return runs
def test_int(self):
packed = pack_int(555)
unpacked = unpack_int(packed)
assert unpacked == 555
assert unpack_int("Banana") == 0
def retry(self, task):
retries = unpack_int(task.context.get("retries"))
if retries < settings.WORKER_RETRY:
log.warning("Queue failed task for re-try...")
task.context["retries"] = retries + 1
task.stage.queue(task.payload, task.context)
def get_stage_counts(cls, crawler, stage):
counts = {}
for level in cls.LEVELS:
key = make_key(crawler, "events", "count", stage, level)
counts[level] = unpack_int(conn.get(key))
return counts
def get(self, slot=None):
key = self._key(slot or self._time())
return unpack_int(self.conn.get(key))
