def test_failed_job_scheduling_with_no_task(self, mock_failed_get_all,
mock_failed_task_delete,
mock_remove_job):
# configure to return entry with max retry count
failed_jobs = fake_failed_jobs.copy()
failed_jobs[0][FailedTask.job_id.name] = uuidutils.generate_uuid()
mock_failed_get_all.return_value = failed_jobs
failed_job = FailedJobHandler(context.get_admin_context())
# call failed job scheduling
failed_job.remove_failed_job(fake_failed_job)
# entry get deleted and job get removed
self.assertEqual(mock_failed_task_delete.call_count, 1)
self.assertEqual(mock_remove_job.call_count, 0)
def test_failed_job_with_job_already_scheduled(self, mock_failed_get_all,
mock_add_job, mock_get_job):
# configure to return entry with job id
failed_jobs = fake_failed_jobs.copy()
failed_jobs[0][FailedTask.job_id.name] = uuidutils.generate_uuid()
mock_failed_get_all.return_value = failed_jobs
# configure to have job in scheduler
mock_get_job.return_value = failed_jobs
failed_job = FailedJobHandler(context.get_admin_context())
# call failed job scheduling
failed_job.remove_failed_job(fake_failed_job['id'])
# the job will not be scheduled
self.assertEqual(mock_add_job.call_count, 0)
def test_failed_job_with_max_retry(self, mock_failed_get_all,
mock_failed_task_delete, mock_get_job,
mock_remove_job):
# configure to return entry with max retry count
failed_jobs = fake_failed_jobs.copy()
failed_jobs[0][FailedTask.retry_count.name] = \
TelemetryCollection.MAX_FAILED_JOB_RETRY_COUNT
mock_failed_get_all.return_value = failed_jobs
failed_job = FailedJobHandler(context.get_admin_context())
# call failed job scheduling
failed_job.schedule_failed_job(failed_jobs[0])
mock_get_job.return_value = True
# entry get deleted and job get removed
self.assertEqual(mock_failed_task_delete.call_count, 1)
self.assertEqual(mock_remove_job.call_count, 1)
def assign_failed_job(self, context, failed_task_id, executor):
if not self.enable_sub_process:
instance = FailedJobHandler.get_instance(context, failed_task_id)
instance.schedule_failed_job(failed_task_id)
else:
if not self.watch_job_id:
self.init_watchers(executor)
local_executor = self.get_local_executor(context, None,
failed_task_id, executor)
self.rpcapi.assign_failed_job_local(context, failed_task_id,
local_executor)
def remove_failed_job(self, context, failed_task_id, executor):
if not self.enable_sub_process:
instance = FailedJobHandler.get_instance(context, failed_task_id)
instance.remove_failed_job(failed_task_id)
else:
job = db.failed_task_get(context, failed_task_id)
storage_id = job['storage_id']
for name in self.executor_map.keys():
if storage_id in self.executor_map[name]["storages"]:
local_executor = "{0}:{1}".format(executor, name)
self.rpcapi.remove_failed_job_local(
context, failed_task_id, local_executor)
tasks, failed_tasks = self.get_all_tasks(storage_id)
if len(failed_tasks) == 0 and len(tasks) == 0:
self.stop_executor(name, local_executor, storage_id)
def test_failed_job_scheduling(self, mock_add_job):
failed_job = FailedJobHandler(context.get_admin_context())
# call failed job scheduling
failed_job.schedule_failed_job(fake_failed_job['id'])
self.assertEqual(mock_add_job.call_count, 1)
def remove_failed_job_local(self, context, failed_task_id):
instance = FailedJobHandler.get_instance(context, failed_task_id)
instance.remove_failed_job(failed_task_id)
def assign_failed_job_local(self, context, failed_task_id):
instance = FailedJobHandler.get_instance(context, failed_task_id)
instance.schedule_failed_job(failed_task_id)