def get_status(self): """Get the current status.""" keys = (self.pending_key, self.running_key, self.finished_key) pending, running, finished = self.conn.mget(keys) return { 'job_id': self.job.id, 'stage': self.stage, 'pending': max(0, unpack_int(pending)), 'running': max(0, unpack_int(running)), 'finished': max(0, unpack_int(finished)), }
def get_status(self): """Get the current status.""" keys = (self.pending_key, self.running_key, self.finished_key) pending, running, finished = self.conn.mget(keys) return { "job_id": self.job.id, "stage": self.stage, "pending": max(0, unpack_int(pending)), "running": max(0, unpack_int(running)), "finished": max(0, unpack_int(finished)), }
def operation_end(cls, crawler, run_id): conn.set(make_key(crawler, "last_run"), pack_now(), ex=REDIS_LONG) pending = conn.decr(make_key(crawler, "run", run_id)) if unpack_int(pending) == 0: conn.set(make_key(crawler, "run", run_id, "end"), pack_now(), ex=REDIS_LONG)
def op_count(cls, crawler, stage=None): """Total operations performed for this crawler""" if stage: total_ops = conn.get(make_key(crawler, stage)) else: total_ops = conn.get(make_key(crawler, "total_ops")) return unpack_int(total_ops)
def runs(cls, crawler): runs = [] for run_id in cls.run_ids(crawler): start = conn.get(make_key(crawler, "run", run_id, "start")) end = conn.get(make_key(crawler, "run", run_id, "end")) total_ops = conn.get(make_key(crawler, "run", run_id, "total_ops")) runs.append({ "run_id": run_id, "total_ops": unpack_int(total_ops), "start": unpack_datetime(start, datetime.utcnow()), "end": unpack_datetime(end), }) return runs
def test_int(self): packed = pack_int(555) unpacked = unpack_int(packed) assert unpacked == 555 assert unpack_int("Banana") == 0
def retry(self, task): retries = unpack_int(task.context.get("retries")) if retries < settings.WORKER_RETRY: log.warning("Queue failed task for re-try...") task.context["retries"] = retries + 1 task.stage.queue(task.payload, task.context)
def get_stage_counts(cls, crawler, stage): counts = {} for level in cls.LEVELS: key = make_key(crawler, "events", "count", stage, level) counts[level] = unpack_int(conn.get(key)) return counts
def get(self, slot=None): key = self._key(slot or self._time()) return unpack_int(self.conn.get(key))