def test_get_unfinished_executions(client):
with client.application.app_context():
app = client.application
app.redis.flushall()
for status in [
JobExecution.Status.enqueued,
JobExecution.Status.pulling,
JobExecution.Status.running,
JobExecution.Status.done,
JobExecution.Status.failed,
]:
_, job, execution = JobExecutionFixture.new_defaults()
execution.status = status
job.save()
topic = Job.get_unfinished_executions()
expect(topic).to_length(2)
for (_, execution) in topic:
expect(execution).to_be_instance_of(JobExecution)
expect(
execution.status
in [JobExecution.Status.pulling, JobExecution.Status.running]
).to_be_true()
def enqueue_missing_monitor_jobs(app):
lock = app.redis.lock(
"EnqueueMissingMonitorJobs",
timeout=7,
sleep=0.2,
blocking_timeout=500,
thread_local=False,
)
if not lock.acquire():
app.logger.info(
"Lock could not be acquired. Trying to enqueue missing monitor jobs later."
)
return
try:
# find running/created executions
executions = Job.get_unfinished_executions(app)
queue = app.monitor_queue
executions_to_monitor = []
for (job, execution) in executions:
if "enqueued_id" in job.metadata and queue.is_scheduled(
job.metadata["enqueued_id"]
):
continue
executions_to_monitor.append((job, execution))
if not executions_to_monitor:
return
current_app.logger.info(
"Found executions missing monitoring. Enqueueing monitor.",
executions=len(executions_to_monitor),
)
# enqueue if execution not scheduled to be monitored
for (job, execution) in executions_to_monitor:
current_app.monitor_queue.enqueue_in(
"5s",
Categories.Monitor,
job.task.task_id,
job.job_id,
execution.execution_id,
)
finally:
try:
lock.release()
except Exception as err:
current_app.logger.error("Lock release error", error=err)