def __call__(self, process_spec):
pid = process_spec['pid']
if not self.time_to_check(pid):
return True
all_workers = Worker.all(connection=rq_redis_connection)
worker = [
w for w in all_workers
if w.hostname == socket.gethostname().encode() and w.pid == pid
].pop()
is_busy = worker.get_state() == WorkerStatus.BUSY
time_since_seen = datetime.datetime.utcnow() - worker.last_heartbeat
seen_lately = time_since_seen.seconds < 60
total_jobs_in_watched_queues = sum(
[len(q.jobs) for q in worker.queues])
has_nothing_to_do = total_jobs_in_watched_queues == 0
is_healthy = is_busy or seen_lately or has_nothing_to_do
self._log(
"Worker %s healthcheck: Is busy? %s. "
"Seen lately? %s (%d seconds ago). "
"Has nothing to do? %s (%d jobs in watched queues). "
"==> Is healthy? %s", worker.key, is_busy, seen_lately,
time_since_seen.seconds, has_nothing_to_do,
total_jobs_in_watched_queues, is_healthy)
return is_healthy
def healthcheck():
hostname = socket.gethostname()
with Connection(rq_redis_connection):
all_workers = Worker.all()
local_workers = [w for w in all_workers if w.hostname == hostname]
heartbeats = [w.last_heartbeat for w in local_workers]
time_since_seen = [
datetime.datetime.utcnow() - hb for hb in heartbeats
]
active = [t.seconds < 60 for t in time_since_seen]
sys.exit(int(not all(active)))