def _new_rq_worker() -> Worker:
return SimpleWorker(
queues=queue_names(),
connection=redis_connection(),
queue_class=Queue, # (1)
job_class=Job, # (2)
exception_handlers=[RqExcMan.exc_handler]) # (3)
def _new_telemetry_worker(monitoring_dir: str) -> Worker:
return TelemetryWorker(
monitoring_dir=monitoring_dir,
queues=queue_names(),
connection=redis_connection(),
queue_class=Queue, # (1)
job_class=Job, # (2)
exception_handlers=[RqExcMan.exc_handler]) # (3)
def find_job_ids_in_registry(rq_reg_key: str, job_id_matcher: str) \
-> Iterable[RqJobId]:
"""
Iterate RQ job IDs found in the named RQ registry that match the
given Redis expression.
:param rq_reg_key: the name of the RQ registry to scan.
:param job_id_matcher: a Redis pattern to match RQ job IDs.
:return: an generator object to iterate the matching job IDs.
"""
redis = redis_connection()
for jid in redis.zscan_iter(name=rq_reg_key, match=job_id_matcher):
yield jid[0].decode('utf-8')
def find_job_keys(job_id_matcher: str) -> Iterable[RqJobKey]:
"""
Iterate the RQ job keys having a job ID part matching the given Redis
expression.
:param job_id_matcher: a Redis pattern to match the job ID part of an
RQ job key.
:return: an generator object to iterate the matching keys.
"""
key_matcher = job_key_matcher(job_id_matcher)
redis = redis_connection()
for k in redis.scan_iter(match=key_matcher):
yield k.decode('utf-8') # (*)
def load_jobs(job_ids: Iterable[RqJobId]) -> Iterable[Job]:
"""
Iterate RQ jobs having an ID in the input set of ``job_ids``.
Notice that the returned jobs may be less than the number of input job
IDs since other processes acting on the RQ queues may delete or purge
jobs while this fetch operation is in progress.
:param job_ids: the RQ job IDs of the jobs to fetch.
:return: an generator object to iterate the jobs.
"""
redis = redis_connection()
splitter = IterCostSplitter(cost_fn=lambda _: 1, batch_max_cost=100) # (1)
for jid_batch_iter in splitter.iter_batches(job_ids):
jid_batch = list(jid_batch_iter) # (2)
jobs = Job.fetch_many(job_ids=jid_batch, connection=redis)
for j in jobs:
if j is not None: # (3)
yield j
def work_queue(self) -> WorkQ:
"""
:return: the work queue where to put this task.
"""
return Queue(self.queue_name(), connection=redis_connection())
def new_work_q_size_sampler(monitoring_dir: Path) -> WorkQSizeSampler:
q_name = default_queue_name()
q = Queue(q_name, connection=redis_connection())
bucket = _new_bucket(str(monitoring_dir), GAUGE_FILE_PREFIX)
return WorkQSizeSampler(bucket=bucket, q=q, q_name=q_name)