async def delay(
self,
args: Optional[Tuple[Any, ...]] = None,
kwargs: Optional[Dict[str, Any]] = None,
job_id: str = None,
countdown: Union[float, datetime.timedelta] = 0,
eta: Optional[datetime.datetime] = None,
expire: Optional[Union[float, datetime.datetime]] = None,
job_retry: int = 0,
job_retry_after: int = 60,
) -> Job:
if not job_id:
job_id = uuid4().hex
if countdown:
defer_ts = to_ms_timestamp(countdown)
elif eta:
defer_ts = to_ms_timestamp(eta)
else:
defer_ts = timestamp_ms_now()
expire_time = None
expires = expire or self.expire
if expires:
expire_time = ms_to_datetime(to_ms_timestamp(expires))
job = await Job.get_or_none(job_id=job_id)
if job:
logger.warning(f"Job {job_id} exists")
return job
job = Job(
task=self.function.__name__,
args=args,
kwargs=kwargs,
job_retry=job_retry or self.job_retry,
queue=self.queue,
job_id=job_id,
expire_time=expire_time,
enqueue_time=timezone.now(),
job_retry_after=job_retry_after,
)
if not eta and not countdown:
job.status = JobStatus.queued
await job.save()
await self.rearq.redis.xadd(self.queue, {"job_id": job_id})
else:
job.status = JobStatus.deferred
await job.save()
await self.rearq.redis.zadd(DELAY_QUEUE, defer_ts,
f"{self.queue}:{job_id}")
return job
async def run_job(self, queue: str, msg_id: str, job: Job):
if job.expire_time and job.expire_time > timezone.now():
logger.warning(f"job {job.job_id} is expired, ignore")
job.status = JobStatus.expired
await job.save(update_fields=["status"])
return
job_id = job.job_id
job_result = JobResult(
msg_id=msg_id, job=job, worker=self.worker_name, start_time=timezone.now()
)
task = self._task_map.get(job.task)
if not task:
logger.warning(f"job {job_id}, task {job.task} not found")
job_result.result = "task not found"
await job_result.save()
return job_result
ref = f"{job_id}:{job.task}"
start_ms = timestamp_ms_now()
logger.info(
"%6.2fs → %s(%s)%s"
% (
(start_ms - to_ms_timestamp(job.enqueue_time)) / 1000,
ref,
args_to_string(job.args, job.kwargs),
f" try={job.job_retries}" if job.job_retries > 1 else "",
)
)
try:
async with async_timeout.timeout(self.job_timeout):
if task.bind:
result = await task.function(task, *(job.args or []), **(job.kwargs or {}))
else:
result = await task.function(*(job.args or []), **(job.kwargs or {}))
job_result.success = True
job_result.finish_time = timezone.now()
job.status = JobStatus.success
logger.info("%6.2fs ← %s ● %s" % ((timestamp_ms_now() - start_ms) / 1000, ref, result))
self.jobs_complete += 1
except Exception as e:
job_result.finish_time = timezone.now()
self.jobs_failed += 1
result = f"Run task error in NO.{job.job_retries} times, exc: {e}, retry after {self.job_retry_after} seconds"
logger.error("%6.2fs ← %s ● %s" % ((timestamp_ms_now() - start_ms) / 1000, ref, result))
if job.job_retries >= job.job_retry:
t = (timestamp_ms_now() - to_ms_timestamp(job.enqueue_time)) / 1000
logger.error("%6.2fs ! %s max retries %d exceeded" % (t, ref, job.job_retry))
job.status = JobStatus.failed
else:
job.status = JobStatus.deferred
job.job_retries = F("job_retries") + 1
await self.rearq.zadd(to_ms_timestamp(self.job_retry_after), f"{queue}:{job_id}")
finally:
await self._xack(queue, msg_id)
await job.save(update_fields=["status", "job_retries"])
job_result.result = result
await job_result.save()
return job_result
def set_next(self):
self.next_run = to_ms_timestamp(self.crontab.next(default_utc=False))