def test_atomic_update_or_create_does_retry_on_db_operational_error(
self, request, jobstore):
now = timezone.now()
job = DjangoJob.objects.create(id="test_job", next_run_time=now)
request.addfinalizer(job.delete)
ex = DjangoJobExecution.objects.create(
job_id=job.id,
run_time=job.next_run_time - timedelta(seconds=5),
status=DjangoJobExecution.SENT,
)
with mock.patch.object(db.connection, "close") as close_mock:
with pytest.raises(db.OperationalError,
match="Some DB-related error"):
with mock.patch(
"django_apscheduler.models.DjangoJobExecution.objects.select_for_update",
side_effect=conftest.raise_db_operational_error,
):
DjangoJobExecution.atomic_update_or_create(
RLock(),
ex.job_id,
ex.run_time,
DjangoJobExecution.SUCCESS,
)
assert close_mock.call_count == 1
def test_atomic_update_or_create_updates_existing_jobs(
self, request, jobstore):
now = timezone.now()
job = DjangoJob.objects.create(id="test_job", next_run_time=now)
request.addfinalizer(job.delete)
ex = DjangoJobExecution.objects.create(
job_id=job.id,
run_time=job.next_run_time - timedelta(seconds=5),
status=DjangoJobExecution.SENT,
)
assert ex.status == DjangoJobExecution.SENT
assert ex.duration is None
assert ex.finished is None
DjangoJobExecution.atomic_update_or_create(
RLock(),
ex.job_id,
ex.run_time,
DjangoJobExecution.SUCCESS,
)
ex.refresh_from_db()
assert ex.status == DjangoJobExecution.SUCCESS
assert ex.duration is not None
assert ex.finished is not None
def handle_error_event(cls, event: JobExecutionEvent) -> int:
"""
Store failed job execution status in the database.
:param event: JobExecutionEvent instance
:return: DjangoJobExecution ID
"""
if event.code == events.EVENT_JOB_ERROR:
if event.exception:
exception = str(event.exception)
traceback = str(event.traceback)
else:
exception = f"Job '{event.job_id}' raised an error!"
traceback = None
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_time,
DjangoJobExecution.ERROR,
exception=exception,
traceback=traceback,
)
elif event.code in [
events.EVENT_JOB_MAX_INSTANCES, events.EVENT_JOB_MISSED
]:
# Job execution will not have been logged yet - do so now
if event.code == events.EVENT_JOB_MAX_INSTANCES:
status = DjangoJobExecution.MAX_INSTANCES
exception = (
f"Execution of job '{event.job_id}' skipped: maximum number of running "
f"instances reached!")
else:
status = DjangoJobExecution.MISSED
exception = f"Run time of job '{event.job_id}' was missed!"
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_time,
status,
exception=exception,
)
else:
raise NotImplementedError(
f"Don't know how to handle JobExecutionEvent '{event.code}'. Expected "
f"one of '{[events.EVENT_JOB_ERROR, events.EVENT_JOB_MAX_INSTANCES, events.EVENT_JOB_MISSED]}'."
)
return job_execution.id
def test_atomic_update_or_create_creates_new_job(self, request, jobstore):
now = timezone.now()
job = DjangoJob.objects.create(id="test_job", next_run_time=now)
request.addfinalizer(job.delete)
DjangoJobExecution.atomic_update_or_create(
RLock(),
job.id,
job.next_run_time - timedelta(seconds=5),
DjangoJobExecution.SUCCESS,
)
assert DjangoJobExecution.objects.filter(job_id=job.id).exists()
def handle_error_event(cls, event: JobExecutionEvent) -> Union[int, None]:
"""
Store "failed" job execution status in the database.
:param event: JobExecutionEvent instance
:return: DjangoJobExecution ID or None if the job execution could not be logged.
"""
try:
if event.code == events.EVENT_JOB_ERROR:
if event.exception:
exception = str(event.exception)
traceback = str(event.traceback)
else:
exception = f"Job '{event.job_id}' raised an error!"
traceback = None
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_time,
DjangoJobExecution.ERROR,
exception=exception,
traceback=traceback,
)
elif event.code == events.EVENT_JOB_MISSED:
# Job execution will not have been logged yet - do so now
status = DjangoJobExecution.MISSED
exception = f"Run time of job '{event.job_id}' was missed!"
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_time,
status,
exception=exception,
)
else:
raise NotImplementedError(
f"Don't know how to handle JobExecutionEvent '{event.code}'. Expected "
f"one of '{[events.EVENT_JOB_ERROR, events.EVENT_JOB_MAX_INSTANCES, events.EVENT_JOB_MISSED]}'."
)
except IntegrityError:
logger.warning(
f"Job '{event.job_id}' no longer exists! Skipping logging of job execution..."
)
return None
return job_execution.id
def handle_execution_event(cls,
event: JobExecutionEvent) -> Union[int, None]:
"""
Store "successful" job execution status in the database.
:param event: JobExecutionEvent instance
:return: DjangoJobExecution ID or None if the job execution could not be logged.
"""
if event.code != events.EVENT_JOB_EXECUTED:
raise NotImplementedError(
f"Don't know how to handle JobExecutionEvent '{event.code}'. Expected "
f"'{events.EVENT_JOB_EXECUTED}'.")
try:
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_time,
DjangoJobExecution.SUCCESS,
)
except IntegrityError:
logger.warning(
f"Job '{event.job_id}' no longer exists! Skipping logging of job execution..."
)
return None
return job_execution.id
def handle_submission_event(cls, event: JobSubmissionEvent):
"""
Create and return new job execution instance in the database when the job is submitted to the scheduler.
:param event: JobExecutionEvent instance
:return: DjangoJobExecution ID or None if the job execution could not be logged.
"""
try:
if event.code == events.EVENT_JOB_SUBMITTED:
# Start logging a new job execution
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_times[0],
DjangoJobExecution.SENT,
)
elif event.code == events.EVENT_JOB_MAX_INSTANCES:
status = DjangoJobExecution.MAX_INSTANCES
exception = (
f"Execution of job '{event.job_id}' skipped: maximum number of running "
f"instances reached!"
)
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_times[0],
status,
exception=exception,
)
else:
raise NotImplementedError(
f"Don't know how to handle JobSubmissionEvent '{event.code}'. Expected "
f"one of '{[events.EVENT_JOB_SUBMITTED, events.EVENT_JOB_MAX_INSTANCES]}'."
)
except IntegrityError:
logger.warning(
f"Job '{event.job_id}' no longer exists! Skipping logging of job execution..."
)
return None
return job_execution.id
def handle_execution_event(cls, event: JobExecutionEvent) -> int:
"""
Store successful job execution status in the database.
:param event: JobExecutionEvent instance
:return: DjangoJobExecution ID
"""
if event.code != events.EVENT_JOB_EXECUTED:
raise NotImplementedError(
f"Don't know how to handle JobExecutionEvent '{event.code}'. Expected "
f"'{events.EVENT_JOB_EXECUTED}'.")
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock, event.job_id, event.scheduled_run_time,
DjangoJobExecution.SUCCESS)
return job_execution.id
def handle_submission_event(cls, event: JobSubmissionEvent):
"""
Create and return new job execution instance in the database when the job is submitted to the scheduler.
:param event: JobExecutionEvent instance
:return: DjangoJobExecution ID
"""
if event.code != events.EVENT_JOB_SUBMITTED:
raise NotImplementedError(
f"Don't know how to handle JobSubmissionEvent '{event.code}'. Expected "
f"'{events.EVENT_JOB_SUBMITTED}'.")
# Start logging a new job execution
job_execution = DjangoJobExecution.atomic_update_or_create(
cls.lock,
event.job_id,
event.scheduled_run_times[0],
DjangoJobExecution.SENT,
)
return job_execution.id