def test_get_unfinished_executions(client): with client.application.app_context(): app = client.application app.redis.flushall() for status in [ JobExecution.Status.enqueued, JobExecution.Status.pulling, JobExecution.Status.running, JobExecution.Status.done, JobExecution.Status.failed, ]: _, job, execution = JobExecutionFixture.new_defaults() execution.status = status job.save() topic = Job.get_unfinished_executions() expect(topic).to_length(2) for (_, execution) in topic: expect(execution).to_be_instance_of(JobExecution) expect( execution.status in [JobExecution.Status.pulling, JobExecution.Status.running] ).to_be_true()
def enqueue_missing_monitor_jobs(app): lock = app.redis.lock( "EnqueueMissingMonitorJobs", timeout=7, sleep=0.2, blocking_timeout=500, thread_local=False, ) if not lock.acquire(): app.logger.info( "Lock could not be acquired. Trying to enqueue missing monitor jobs later." ) return try: # find running/created executions executions = Job.get_unfinished_executions(app) queue = app.monitor_queue executions_to_monitor = [] for (job, execution) in executions: if "enqueued_id" in job.metadata and queue.is_scheduled( job.metadata["enqueued_id"] ): continue executions_to_monitor.append((job, execution)) if not executions_to_monitor: return current_app.logger.info( "Found executions missing monitoring. Enqueueing monitor.", executions=len(executions_to_monitor), ) # enqueue if execution not scheduled to be monitored for (job, execution) in executions_to_monitor: current_app.monitor_queue.enqueue_in( "5s", Categories.Monitor, job.task.task_id, job.job_id, execution.execution_id, ) finally: try: lock.release() except Exception as err: current_app.logger.error("Lock release error", error=err)