def test_successful_with_full_recipe(self):
"""Tests calling QueueManager.handle_job_completion() successfully with all jobs in a recipe."""
# Queue the recipe
handler = Queue.objects.queue_new_recipe(self.recipe_type, self.data,
self.event)
# Fake out completing Job 1
job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1').job
job_exe_1 = job_test_utils.create_job_exe(job=job_1, status='RUNNING')
output_file_1 = product_test_utils.create_product(
job_exe=job_exe_1, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(
job_exe=job_exe_1, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter('Test Output 1',
[output_file_1.id, output_file_2.id])
job_exe_output_1 = JobExecutionOutput()
job_exe_output_1.job_exe_id = job_exe_1.id
job_exe_output_1.job_id = job_exe_1.job_id
job_exe_output_1.job_type_id = job_exe_1.job_type_id
job_exe_output_1.exe_num = job_exe_1.exe_num
job_exe_output_1.output = results.get_dict()
job_exe_output_1.save()
Job.objects.filter(pk=job_1.id).update(status='RUNNING')
Queue.objects.handle_job_completion(job_1.id, job_1.num_exes, now())
# Fake out completing Job 2
job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2').job
job_exe_2 = job_test_utils.create_job_exe(job=job_2, status='RUNNING')
output_file_1 = product_test_utils.create_product(
job_exe=job_exe_2, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(
job_exe=job_exe_2, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter('Test Output 2',
[output_file_1.id, output_file_2.id])
job_exe_output_2 = JobExecutionOutput()
job_exe_output_2.job_exe_id = job_exe_2.id
job_exe_output_2.job_id = job_exe_2.job_id
job_exe_output_2.job_type_id = job_exe_2.job_type_id
job_exe_output_2.exe_num = job_exe_2.exe_num
job_exe_output_2.output = results.get_dict()
job_exe_output_2.save()
Job.objects.filter(pk=job_2.id).update(status='RUNNING')
# Call method to test
Queue.objects.handle_job_completion(job_2.id, job_2.num_exes, now())
# Make sure final recipe attributes are updated
recipe = Recipe.objects.get(pk=handler.recipe.id)
self.assertIsNotNone(recipe.completed)
def test_process_job_output(self):
"""Tests calling JobManager.process_job_output()"""
output_1 = JobResults()
output_1.add_file_parameter('foo', 1)
output_2 = JobResults()
output_2.add_file_parameter('foo', 2)
# These jobs have completed and have their execution results
job_exe_1 = job_test_utils.create_job_exe(status='COMPLETED',
output=output_1)
job_exe_2 = job_test_utils.create_job_exe(status='COMPLETED',
output=output_2)
# These jobs have their execution results, but have not completed
job_exe_3 = job_test_utils.create_job_exe(status='RUNNING')
job_exe_4 = job_test_utils.create_job_exe(status='RUNNING')
for job_exe in [job_exe_3, job_exe_4]:
job_exe_output = JobExecutionOutput()
job_exe_output.job_exe_id = job_exe.id
job_exe_output.job_id = job_exe.job_id
job_exe_output.job_type_id = job_exe.job.job_type_id
job_exe_output.exe_num = job_exe.exe_num
job_exe_output.output = JobResults().get_dict()
job_exe_output.save()
# These jobs have completed, but do not have their execution results
job_exe_5 = job_test_utils.create_job_exe(status='RUNNING')
job_exe_6 = job_test_utils.create_job_exe(status='RUNNING')
for job in [job_exe_5.job, job_exe_6.job]:
job.status = 'COMPLETED'
job.save()
# Test method
job_ids = [
job_exe.job_id for job_exe in
[job_exe_1, job_exe_2, job_exe_3, job_exe_4, job_exe_5, job_exe_6]
]
result_ids = Job.objects.process_job_output(job_ids, timezone.now())
self.assertEqual(set(result_ids), {job_exe_1.job_id, job_exe_2.job_id})
# Jobs 1 and 2 should have output populated, jobs 3 through 6 should not
jobs = list(Job.objects.filter(id__in=job_ids).order_by('id'))
self.assertEqual(len(jobs), 6)
self.assertTrue(jobs[0].has_output())
self.assertDictEqual(jobs[0].output, output_1.get_dict())
self.assertTrue(jobs[1].has_output())
self.assertDictEqual(jobs[1].output, output_2.get_dict())
self.assertFalse(jobs[2].has_output())
self.assertFalse(jobs[3].has_output())
self.assertFalse(jobs[4].has_output())
self.assertFalse(jobs[5].has_output())
def create_job_exe(job_type=None,
job=None,
exe_num=None,
node=None,
timeout=None,
input_file_size=10.0,
queued=None,
started=None,
status='RUNNING',
error=None,
ended=None,
output=None,
task_results=None):
"""Creates a job_exe model for unit testing, may also create job_exe_end and job_exe_output models depending on
status
:returns: The job_exe model
:rtype: :class:`job.execution.job_exe.RunningJobExecution`
"""
when = timezone.now()
if not job:
job = create_job(job_type=job_type,
status=status,
input_file_size=input_file_size)
job_type = job.job_type
job_exe = JobExecution()
job_exe.job = job
job_exe.job_type = job_type
if not exe_num:
exe_num = job.num_exes
job_exe.exe_num = exe_num
job_exe.set_cluster_id('1234', job.id, job_exe.exe_num)
if not node:
node = node_utils.create_node()
job_exe.node = node
if not timeout:
timeout = job.timeout
job_exe.timeout = timeout
job_exe.input_file_size = input_file_size
job_exe.resources = job.get_resources().get_json().get_dict()
job_exe.configuration = ExecutionConfiguration().get_dict()
if not queued:
queued = when
job_exe.queued = queued
if not started:
started = when + datetime.timedelta(seconds=1)
job_exe.started = started
job_exe.save()
if status in ['COMPLETED', 'FAILED', 'CANCELED']:
job_exe_end = JobExecutionEnd()
job_exe_end.job_exe_id = job_exe.id
job_exe_end.job = job_exe.job
job_exe_end.job_type = job_exe.job_type
job_exe_end.exe_num = job_exe.exe_num
if not task_results:
task_results = TaskResults()
job_exe_end.task_results = task_results.get_dict()
job_exe_end.status = status
if status == 'FAILED' and not error:
error = error_test_utils.create_error()
job_exe_end.error = error
job_exe_end.node = node
job_exe_end.queued = queued
job_exe_end.started = started
job_exe_end.seed_started = task_results.get_task_started('main')
job_exe_end.seed_ended = task_results.get_task_ended('main')
if not ended:
ended = started + datetime.timedelta(seconds=1)
job_exe_end.ended = ended
job_exe_end.save()
if status == 'COMPLETED' or output:
job_exe_output = JobExecutionOutput()
job_exe_output.job_exe_id = job_exe.id
job_exe_output.job = job_exe.job
job_exe_output.job_type = job_exe.job_type
job_exe_output.exe_num = job_exe.exe_num
if not output:
output = JobResults()
job_exe_output.output = output.get_dict()
job_exe_output.save()
return job_exe
def test_successful_supersede(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe."""
# Queue initial recipe and complete its first job
node = node_test_utils.create_node()
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event)
recipe = Recipe.objects.get(id=recipe_id)
recipe_job_1 = RecipeJob.objects.select_related('job__job_exe').get(recipe_id=recipe_id, job_name='Job 1')
job_exe_1 = JobExecution.objects.get(job_id=recipe_job_1.job_id)
queued_job_exe = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_1.id))
queued_job_exe.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe], {})
results = JobResults()
results.add_file_list_parameter('Test Output 1', [product_test_utils.create_product().file_id])
JobExecution.objects.filter(id=job_exe_1.id).update(results=results.get_dict())
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(name=self.job_type_2.name, version='New Version',
interface=self.job_type_2.interface)
new_definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'New Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name': 'New Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 2')
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_recipe_id = Queue.objects.queue_new_recipe(new_recipe_type, None, event, recipe, delta, superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=recipe_id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_recipe_id)
self.assertEqual(new_recipe.superseded_recipe_id, recipe_id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
# Complete both the old and new job 2 and check that only the new recipe completes
job_exe_2 = JobExecution.objects.get(job_id=recipe_job_2.job_id)
queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_2.id))
queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe_2], {})
Queue.objects.handle_job_completion(job_exe_2.id, now())
new_job_exe_2 = JobExecution.objects.get(job_id=new_recipe_job_2.job_id)
new_queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=new_job_exe_2.id))
new_queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000,
disk_total=2000))
Queue.objects.schedule_job_executions('123', [new_queued_job_exe_2], {})
Queue.objects.handle_job_completion(new_job_exe_2.id, now())
recipe = Recipe.objects.get(id=recipe.id)
new_recipe = Recipe.objects.get(id=new_recipe.id)
self.assertIsNone(recipe.completed)
self.assertIsNotNone(new_recipe.completed)
def test_successful_job_1_completed(self, mock_store):
'''Tests calling RecipeDefinition.get_next_jobs_to_queue() successfully when job 1 has been completed.'''
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}]
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
'workspace_id': 1,
}
recipe_data = RecipeData(data)
recipe_definition.validate_data(recipe_data)
png_file_ids = [98, 99, 100]
job_results = JobResults()
job_results.add_file_list_parameter(self.output_name_1, png_file_ids)
job_1 = Job.objects.select_related('job_type').get(pk=self.job_1.id)
job_1.results = job_results.get_dict()
job_1.save()
job_2 = Job.objects.select_related('job_type').get(pk=self.job_2.id)
results = recipe_definition.get_next_jobs_to_queue(
recipe_data, {'Job 2': job_2}, {'Job 1': job_1})
# Make sure only Job 2 is returned and that its job data is correct
self.assertListEqual([self.job_2.id], results.keys())
self.assertDictEqual(
results[self.job_2.id].get_dict(), {
'version':
'1.0',
'input_data': [{
'name': self.input_name_2,
'file_ids': png_file_ids,
}],
'output_data': [{
'name': self.output_name_2,
'workspace_id': 1,
}],
})
def test_successful_job_1_completed(self, mock_store):
'''Tests calling RecipeDefinition.get_next_jobs_to_queue() successfully when job 1 has been completed.'''
definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
'workspace_id': 1,
}
recipe_data = RecipeData(data)
recipe_definition.validate_data(recipe_data)
png_file_ids = [98, 99, 100]
job_results = JobResults()
job_results.add_file_list_parameter(self.output_name_1, png_file_ids)
job_1 = Job.objects.select_related('job_type').get(pk=self.job_1.id)
job_1.results = job_results.get_dict()
job_1.save()
job_2 = Job.objects.select_related('job_type').get(pk=self.job_2.id)
results = recipe_definition.get_next_jobs_to_queue(recipe_data, {'Job 2': job_2}, {'Job 1': job_1})
# Make sure only Job 2 is returned and that its job data is correct
self.assertListEqual([self.job_2.id], results.keys())
self.assertDictEqual(results[self.job_2.id].get_dict(), {
'version': '1.0',
'input_data': [{
'name': self.input_name_2,
'file_ids': png_file_ids,
}],
'output_data': [{
'name': self.output_name_2,
'workspace_id': 1,
}],
})
