def test_execute_with_recipe(self):
"""Tests calling PurgeJobs.execute() successfully with job as part of recipe"""
recipe = recipe_test_utils.create_recipe()
job_exe = job_test_utils.create_job_exe(status='COMPLETED')
job = job_exe.job
recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job, save=True)
source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id, trigger_event=trigger)
# Add job to message
message = PurgeJobs()
message._purge_job_ids = [job.id]
message.trigger_id = trigger.id
message.source_file_id = source_file.id
message.status_change = timezone.now()
# Execute message
result = message.execute()
self.assertTrue(result)
# Check that a new message to purge source file was created
msgs = [msg for msg in message.new_messages if msg.type == 'purge_recipe']
self.assertEqual(len(msgs), 1)
for msg in msgs:
self.assertEqual(msg.recipe_id, recipe.id)
def test_execute_with_parent_recipe(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create recipes
recipe = recipe_test_utils.create_recipe()
parent_recipe = recipe_test_utils.create_recipe()
recipe_test_utils.create_recipe_node(recipe=parent_recipe,
node_name='A',
sub_recipe=recipe,
save=True)
# Create message
message = create_purge_recipe_message(recipe_id=recipe.id,
trigger_id=self.trigger.id,
source_file_id=self.file_1.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the parent recipe was sent
msgs = [
msg for msg in message.new_messages if msg.type == 'purge_recipe'
]
self.assertEqual(len(msgs), 1)
for msg in msgs:
self.assertEqual(msg.recipe_id, parent_recipe.id)
# Assert models were deleted
self.assertEqual(Recipe.objects.filter(id=recipe.id).count(), 0)
self.assertEqual(RecipeNode.objects.filter(recipe=recipe).count(), 0)
def test_json(self):
"""Tests converting a ProcessCondition message to and from JSON"""
definition = RecipeDefinition(Interface())
# TODO: once DataFilter is implemented, create a DataFilter object here that accepts the inputs
definition.add_condition_node('node_a', Interface(), DataFilter(True))
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
condition=condition,
save=True)
# Create message
message = create_process_condition_messages([condition.id])[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessCondition.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipe')
self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id)
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
def test_get_nodes_to_create(self):
"""Tests calling Recipe.get_nodes_to_create()"""
job_type = job_test_utils.create_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type()
# Create recipe
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_condition_node('B', Interface(), DataFilter(True))
definition.add_condition_node('C', Interface(), DataFilter(True))
definition.add_condition_node('D', Interface(), DataFilter(False))
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_recipe_node('H', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_dependency('A', 'D')
definition.add_dependency('A', 'E')
definition.add_dependency('B', 'E')
definition.add_dependency('B', 'F')
definition.add_dependency('C', 'F')
definition.add_dependency('D', 'G')
definition.add_dependency('E', 'G')
definition.add_dependency('E', 'H')
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
# Nodes A, B, and D already exist
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
save=True)
condition_b = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=True, save=False)
condition_d = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=False)
RecipeCondition.objects.bulk_create([condition_b, condition_d])
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', condition=condition_b, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', condition=condition_d, save=False)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_d])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
nodes_to_create = recipe_instance.get_nodes_to_create()
self.assertSetEqual(set(nodes_to_create.keys()), {'C', 'E', 'H'})
def test_json(self):
"""Tests converting a ProcessCondition message to and from JSON"""
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}])
definition = RecipeDefinition(cond_interface_1)
definition.add_condition_node('node_a', cond_interface_1, df1)
definition.add_recipe_input_connection('node_a', 'cond_int',
'cond_int')
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_dict)
data_1 = Data()
data_1.add_value(JsonValue('cond_int', 0))
data_1_dict = convert_data_to_v6_json(data_1).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_1_dict)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
condition=condition,
save=True)
# Create message
message = create_process_condition_messages([condition.id])[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessCondition.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipe')
self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id)
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
def setUp(self):
django.setup()
# self.recipe_type = recipe_test_utils.create_recipe_type_v6(definition=recipe_test_utils.RECIPE_DEFINITION)
self.recipe = recipe_test_utils.create_recipe()
job_type_1 = job_test_utils.create_seed_job_type()
job_1 = job_test_utils.create_job(job_type=job_type_1)
job_type_2 = job_test_utils.create_seed_job_type()
job_2 = job_test_utils.create_job(job_type=job_type_2)
job_type_3 = job_test_utils.create_seed_job_type()
job_3 = job_test_utils.create_job(job_type=job_type_3)
self.recipe_node1 = recipe_test_utils.create_recipe_node(recipe=self.recipe, node_name='job-1', job=job_1)
self.recipe_node2 = recipe_test_utils.create_recipe_node(recipe=self.recipe, node_name='job-2', job=job_2)
self.recipe_node3 = recipe_test_utils.create_recipe_node(recipe=self.recipe, node_name='job-3', job=job_3)
RecipeNode.objects.bulk_create([self.recipe_node1, self.recipe_node2, self.recipe_node3])
def test_json(self):
"""Tests coverting a UpdateRecipeMetrics message to and from JSON"""
recipe = recipe_test_utils.create_recipe()
job_1 = job_test_utils.create_job(status='FAILED')
job_2 = job_test_utils.create_job(status='CANCELED')
job_3 = job_test_utils.create_job(status='BLOCKED')
job_4 = job_test_utils.create_job(status='BLOCKED')
job_5 = job_test_utils.create_job(status='COMPLETED')
recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_1)
recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_2)
recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_3)
recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_4)
recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_5)
RecipeNode.objects.bulk_create([
recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4,
recipe_node_5
])
# Add recipe to message
message = UpdateRecipeMetrics()
if message.can_fit_more():
message.add_recipe(recipe.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipeMetrics.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(recipe.jobs_total, 5)
self.assertEqual(recipe.jobs_pending, 0)
self.assertEqual(recipe.jobs_blocked, 2)
self.assertEqual(recipe.jobs_queued, 0)
self.assertEqual(recipe.jobs_running, 0)
self.assertEqual(recipe.jobs_failed, 1)
self.assertEqual(recipe.jobs_completed, 1)
self.assertEqual(recipe.jobs_canceled, 1)
def test_convert_recipe_to_v6_json(self):
"""Tests calling convert_recipe_to_v6_json() successfully"""
job_type_1 = job_test_utils.create_seed_job_type()
job_type_2 = job_test_utils.create_seed_job_type()
job_type_3 = job_test_utils.create_seed_job_type()
job_type_4 = job_test_utils.create_seed_job_type()
recipe_type_1 = recipe_test_utils.create_recipe_type_v6()
interface = Interface()
interface.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_1', 'object'))
df1 = DataFilter(filter_list=[{'name': 'file_param_1', 'type': 'media-type', 'condition': '==', 'values': ['image/gif']},
{'name': 'json_param_1', 'type': 'object', 'condition': 'superset of', 'values': [{}]}],
all=False)
definition = RecipeDefinition(interface)
definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num)
definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num)
definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num)
definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num)
definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num)
definition.add_condition_node('F', interface, df1) #False
definition.add_job_node('G', job_type_4.name, job_type_4.version, job_type_4.revision_num)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_dependency('A', 'F')
definition.add_dependency('F', 'G')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_1')
definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1')
recipe = recipe_test_utils.create_recipe()
job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False)
job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False)
job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False)
job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False)
Job.objects.bulk_create([job_a, job_b, job_c, job_e])
condition_f = recipe_test_utils.create_recipe_condition(is_processed=True, is_accepted=False, save=True)
recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', sub_recipe=recipe_d,
save=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', condition=condition_f,
save=False)
recipe_nodes = [recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f]
recipe_instance = RecipeInstance(definition, recipe, recipe_nodes)
json = convert_recipe_to_v6_json(recipe_instance)
RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E', 'F'})
def test_execute_with_sub_recipe(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create recipes
sub_recipe_type = recipe_test_utils.create_recipe_type()
definition = RecipeDefinition(Interface())
definition.add_recipe_node('A', sub_recipe_type.name,
sub_recipe_type.revision_num)
recipe_a = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_a.jobs_completed = 3
recipe_a.jobs_running = 2
recipe_a.jobs_total = 5
Recipe.objects.bulk_create([recipe_a])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='A', sub_recipe=recipe_a, save=False)
RecipeNode.objects.bulk_create([recipe_node_a])
# Create message
message = create_purge_recipe_message(recipe_id=recipe.id,
trigger_id=self.trigger.id,
source_file_id=self.file_1.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the parent recipe was sent
msgs = [
msg for msg in message.new_messages if msg.type == 'purge_recipe'
]
self.assertEqual(len(msgs), 1)
for msg in msgs:
self.assertEqual(msg.recipe_id, recipe_node_a.sub_recipe.id)
def test_execute_with_recipe_legacy(self):
"""Tests calling ProcessRecipeInput.execute() successfully when a legacy sub-recipe has to get its data from its
recipe
"""
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace,
file_size=104857600.0)
file_2 = storage_test_utils.create_file(workspace=workspace,
file_size=987654321.0)
file_3 = storage_test_utils.create_file(workspace=workspace,
file_size=65456.0)
file_4 = storage_test_utils.create_file(workspace=workspace,
file_size=24564165456.0)
manifest_a = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-a',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [],
'json': []
},
'outputs': {
'files': [{
'name': 'output_a',
'pattern': '*.png'
}]
}
}
}
}
job_type_a = job_test_utils.create_job_type(interface=manifest_a)
output_data_a = Data()
output_data_a.add_value(FileValue('output_a', [file_1.id]))
output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict()
manifest_b = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-b',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [],
'json': []
},
'outputs': {
'files': [{
'name': 'output_b',
'pattern': '*.png',
'multiple': True
}]
}
}
}
}
job_type_b = job_test_utils.create_job_type(interface=manifest_b)
output_data_b = Data()
output_data_b.add_value(
FileValue('output_b', [file_2.id, file_3.id, file_4.id]))
output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict()
job_a = job_test_utils.create_job(job_type=job_type_a,
num_exes=1,
status='COMPLETED',
output=output_data_a_dict)
job_b = job_test_utils.create_job(job_type=job_type_b,
num_exes=1,
status='COMPLETED',
output=output_data_b_dict)
sub_recipe_interface_c = Interface()
sub_recipe_interface_c.add_parameter(
FileParameter('input_a', ['image/png']))
sub_recipe_interface_c.add_parameter(
FileParameter('input_b', ['image/png'], multiple=True))
sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c)
sub_recipe_def_dict_c = convert_recipe_definition_to_v1_json(
sub_recipe_def_c).get_dict()
sub_recipe_type_c = recipe_test_utils.create_recipe_type(
definition=sub_recipe_def_dict_c)
sub_recipe_c = recipe_test_utils.create_recipe(
recipe_type=sub_recipe_type_c)
definition = RecipeDefinition(Interface())
definition.add_job_node('node_a', job_type_a.name, job_type_a.version,
job_type_a.revision_num)
definition.add_job_node('node_b', job_type_b.name, job_type_b.version,
job_type_b.revision_num)
definition.add_recipe_node('node_c', sub_recipe_type_c.name,
sub_recipe_type_c.revision_num)
definition.add_dependency('node_c', 'node_a')
definition.add_dependency_input_connection('node_c', 'input_a',
'node_a', 'output_a')
definition.add_dependency('node_c', 'node_b')
definition.add_dependency_input_connection('node_c', 'input_b',
'node_b', 'output_b')
def_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict)
recipe_data_dict = {
'version': '1.0',
'input_data': [],
'workspace_id': workspace.id
}
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=recipe_data_dict)
recipe_node_a = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_a', job=job_a)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_b', job=job_b)
recipe_node_c = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_c])
job_a.recipe = recipe
job_a.save()
job_b.recipe = recipe
job_b.save()
sub_recipe_c.recipe = recipe
sub_recipe_c.save()
# Create message
message = ProcessRecipeInput()
message.recipe_id = sub_recipe_c.id
# Execute message
result = message.execute()
self.assertTrue(result)
sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id)
# Check for update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Check sub-recipe for expected input_file_size
self.assertEqual(sub_recipe_c.input_file_size, 24469.0)
# Check sub-recipe for expected input data
self.assertEqual(
sub_recipe_c.input['version'],
'1.0') # Should be legacy input data with workspace ID
self.assertEqual(sub_recipe_c.input['workspace_id'], workspace.id)
self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()),
{'input_a', 'input_b'})
self.assertListEqual(
sub_recipe_c.get_input_data().values['input_a'].file_ids,
[file_1.id])
self.assertListEqual(
sub_recipe_c.get_input_data().values['input_b'].file_ids,
[file_2.id, file_3.id, file_4.id])
# Make sure sub-recipe input file models are created
input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
self.assertEqual(len(input_files), 4)
file_ids = {input_file.input_file_id for input_file in input_files}
self.assertSetEqual(file_ids,
{file_1.id, file_2.id, file_3.id, file_4.id})
# Test executing message again
message_json_dict = message.to_json()
message = ProcessRecipeInput.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Still should have update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Make sure recipe input file models are unchanged
input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
self.assertEqual(len(input_files), 4)
def test_execute_with_sub_recipes(self):
"""Tests calling UpdateBatchMetrics.execute() successfully with sub-recipes"""
batch = batch_test_utils.create_batch()
recipe_1 = recipe_test_utils.create_recipe(batch=batch)
recipe_2 = recipe_test_utils.create_recipe(batch=batch)
# Recipe 1 jobs
job_1 = job_test_utils.create_job(status='FAILED', save=False)
job_2 = job_test_utils.create_job(status='CANCELED', save=False)
job_3 = job_test_utils.create_job(status='BLOCKED', save=False)
job_4 = job_test_utils.create_job(status='BLOCKED', save=False)
job_5 = job_test_utils.create_job(status='COMPLETED', save=False)
# Recipe 2 jobs
job_6 = job_test_utils.create_job(status='COMPLETED', save=False)
job_7 = job_test_utils.create_job(status='COMPLETED', save=False)
job_8 = job_test_utils.create_job(status='RUNNING', save=False)
job_9 = job_test_utils.create_job(status='QUEUED', save=False)
job_10 = job_test_utils.create_job(status='PENDING', save=False)
job_11 = job_test_utils.create_job(status='PENDING', save=False)
job_12 = job_test_utils.create_job(status='PENDING', save=False)
job_13 = job_test_utils.create_job(status='CANCELED', save=False)
job_14 = job_test_utils.create_job(status='BLOCKED', save=False)
Job.objects.bulk_create([
job_1, job_2, job_3, job_4, job_5, job_6, job_7, job_8, job_9,
job_10, job_11, job_12, job_13, job_14
])
# Recipe 1 sub-recipes
sub_recipe_1 = recipe_test_utils.create_recipe(save=False)
sub_recipe_1.recipe = recipe_1
sub_recipe_1.jobs_total = 26
sub_recipe_1.jobs_pending = 3
sub_recipe_1.jobs_blocked = 4
sub_recipe_1.jobs_queued = 5
sub_recipe_1.jobs_running = 1
sub_recipe_1.jobs_failed = 2
sub_recipe_1.jobs_completed = 3
sub_recipe_1.jobs_canceled = 8
sub_recipe_1.is_completed = False
sub_recipe_2 = recipe_test_utils.create_recipe(save=False)
sub_recipe_2.recipe = recipe_1
sub_recipe_2.jobs_total = 30
sub_recipe_2.jobs_completed = 30
sub_recipe_2.is_completed = True
# Recipe 2 sub-recipes
sub_recipe_3 = recipe_test_utils.create_recipe(save=False)
sub_recipe_3.recipe = recipe_2
sub_recipe_3.jobs_total = 21
sub_recipe_3.jobs_pending = 2
sub_recipe_3.jobs_blocked = 5
sub_recipe_3.jobs_queued = 0
sub_recipe_3.jobs_running = 3
sub_recipe_3.jobs_failed = 2
sub_recipe_3.jobs_completed = 8
sub_recipe_3.jobs_canceled = 1
sub_recipe_3.is_completed = False
sub_recipe_4 = recipe_test_utils.create_recipe(save=False)
sub_recipe_4.recipe = recipe_2
sub_recipe_4.jobs_total = 7
sub_recipe_4.jobs_completed = 7
sub_recipe_4.is_completed = True
sub_recipe_5 = recipe_test_utils.create_recipe(save=False)
sub_recipe_5.recipe = recipe_2
sub_recipe_5.jobs_total = 12
sub_recipe_5.jobs_completed = 12
sub_recipe_5.is_completed = True
Recipe.objects.bulk_create([
sub_recipe_1, sub_recipe_2, sub_recipe_3, sub_recipe_4,
sub_recipe_5
])
# Recipe 1 nodes
recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_1)
recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_2)
recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_3)
recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_4)
recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_5)
recipe_node_6 = recipe_test_utils.create_recipe_node(
recipe=recipe_1, sub_recipe=sub_recipe_1)
recipe_node_7 = recipe_test_utils.create_recipe_node(
recipe=recipe_1, sub_recipe=sub_recipe_2)
# Recipe 2 nodes
recipe_node_8 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_6)
recipe_node_9 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_7)
recipe_node_10 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_8)
recipe_node_11 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_9)
recipe_node_12 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_10)
recipe_node_13 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_11)
recipe_node_14 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_12)
recipe_node_15 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_13)
recipe_node_16 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_14)
recipe_node_17 = recipe_test_utils.create_recipe_node(
recipe=recipe_2, sub_recipe=sub_recipe_3)
recipe_node_18 = recipe_test_utils.create_recipe_node(
recipe=recipe_2, sub_recipe=sub_recipe_4)
recipe_node_19 = recipe_test_utils.create_recipe_node(
recipe=recipe_2, sub_recipe=sub_recipe_5)
RecipeNode.objects.bulk_create([
recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4,
recipe_node_5, recipe_node_6, recipe_node_7, recipe_node_8,
recipe_node_9, recipe_node_10, recipe_node_11, recipe_node_12,
recipe_node_13, recipe_node_14, recipe_node_15, recipe_node_16,
recipe_node_17, recipe_node_18, recipe_node_19
])
# Generate recipe metrics
Recipe.objects.update_recipe_metrics([
sub_recipe_1.id, sub_recipe_2.id, sub_recipe_3.id, sub_recipe_4.id,
sub_recipe_5.id
])
Recipe.objects.update_recipe_metrics([recipe_1.id, recipe_2.id])
# Add batch to message
message = UpdateBatchMetrics()
if message.can_fit_more():
message.add_batch(batch.id)
# Execute message
result = message.execute()
self.assertTrue(result)
batch = Batch.objects.get(id=batch.id)
self.assertEqual(batch.jobs_total, 110)
self.assertEqual(batch.jobs_pending, 8)
self.assertEqual(batch.jobs_blocked, 12)
self.assertEqual(batch.jobs_queued, 6)
self.assertEqual(batch.jobs_running, 5)
self.assertEqual(batch.jobs_failed, 5)
self.assertEqual(batch.jobs_completed, 63)
self.assertEqual(batch.jobs_canceled, 11)
self.assertEqual(batch.recipes_total, 7)
self.assertEqual(batch.recipes_completed, 3)
# Test executing message again
message_json_dict = message.to_json()
message = UpdateBatchMetrics.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
def test_execute(self):
"""Tests calling ProcessCondition.execute() successfully"""
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace,
file_size=104857600.0)
file_2 = storage_test_utils.create_file(workspace=workspace,
file_size=987654321.0)
file_3 = storage_test_utils.create_file(workspace=workspace,
file_size=65456.0)
file_4 = storage_test_utils.create_file(workspace=workspace,
file_size=24564165456.0)
manifest_1 = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-a',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [],
'json': []
},
'outputs': {
'files': [{
'name': 'OUTPUT_A',
'pattern': '*.png',
'multiple': True
}]
}
}
}
}
job_type_1 = job_test_utils.create_job_type(interface=manifest_1)
manifest_2 = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-b',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': []
},
'outputs': {
'files': [{
'name': 'OUTPUT_B',
'pattern': '*.png',
'multiple': True
}]
}
}
}
}
job_type_2 = job_test_utils.create_job_type(interface=manifest_2)
output_1_dict = {
'version':
'1.0',
'output_data': [{
'name': 'OUTPUT_A',
'file_ids': [file_1.id, file_2.id]
}]
}
output_2_dict = {
'version':
'1.0',
'output_data': [{
'name': 'OUTPUT_B',
'file_ids': [file_3.id, file_4.id]
}]
}
cond_interface = Interface()
cond_interface.add_parameter(
FileParameter('INPUT_C_1', [], multiple=True))
cond_interface.add_parameter(
FileParameter('INPUT_C_2', [], multiple=True))
definition = RecipeDefinition(Interface())
definition.add_job_node('node_a', job_type_1.name, job_type_1.version,
job_type_1.revision_num)
definition.add_job_node('node_b', job_type_2.name, job_type_2.version,
job_type_2.revision_num)
# TODO: once DataFilter is implemented, create a DataFilter object here that accepts the inputs
definition.add_condition_node('node_c', cond_interface,
DataFilter(True))
definition.add_dependency('node_a', 'node_c')
definition.add_dependency('node_b', 'node_c')
definition.add_dependency_input_connection('node_c', 'INPUT_C_1',
'node_a', 'OUTPUT_A')
definition.add_dependency_input_connection('node_c', 'INPUT_C_2',
'node_b', 'OUTPUT_B')
def_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict)
recipe_data_dict = {
'version': '1.0',
'input_data': [],
'workspace_id': workspace.id
}
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=recipe_data_dict)
job_1 = job_test_utils.create_job(job_type=job_type_1,
num_exes=1,
status='COMPLETED',
output=output_1_dict,
recipe=recipe)
job_2 = job_test_utils.create_job(job_type=job_type_2,
num_exes=1,
status='COMPLETED',
output=output_2_dict,
recipe=recipe)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
job=job_1,
save=False)
node_b = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_b',
job=job_2,
save=False)
node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_c',
condition=condition,
save=False)
RecipeNode.objects.bulk_create([node_a, node_b, node_c])
# Create message
message = create_process_condition_messages([condition.id])[0]
# Execute message
result = message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
# Check for update_recipe message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipe')
self.assertEqual(message.new_messages[0].root_recipe_id, recipe.id)
# Check condition flags
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
# Check condition for expected data
self.assertSetEqual(set(condition.get_data().values.keys()),
{'INPUT_C_1', 'INPUT_C_2'})
self.assertListEqual(condition.get_data().values['INPUT_C_1'].file_ids,
[file_1.id, file_2.id])
self.assertListEqual(condition.get_data().values['INPUT_C_2'].file_ids,
[file_3.id, file_4.id])
# Test executing message again
message_json_dict = message.to_json()
message = ProcessCondition.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Still should have update_recipe message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipe')
self.assertEqual(message.new_messages[0].root_recipe_id, recipe.id)
def test_execute_with_recipe(self):
"""Tests calling CreateJobs.execute() successfully with a recipe that supersedes another recipe"""
from batch.test import utils as batch_test_utils
from recipe.models import RecipeNode
from recipe.test import utils as recipe_test_utils
job_type = job_test_utils.create_seed_job_type()
recipe_type = recipe_test_utils.create_recipe_type()
superseded_recipe = recipe_test_utils.create_recipe(
recipe_type=recipe_type, is_superseded=True)
node_name = 'recipe_node'
superseded_job = job_test_utils.create_job(job_type=job_type,
is_superseded=True)
recipe_test_utils.create_recipe_node(recipe=superseded_recipe,
node_name=node_name,
job=superseded_job,
save=True)
batch = batch_test_utils.create_batch()
recipe = recipe_test_utils.create_recipe(
recipe_type=recipe_type,
superseded_recipe=superseded_recipe,
batch=batch)
# Create and execute message
message = create_jobs_message_for_recipe(recipe,
node_name,
job_type.name,
job_type.version,
job_type.revision_num,
process_input=True)
result = message.execute()
self.assertTrue(result)
new_job = Job.objects.get(job_type_id=job_type.id, recipe_id=recipe.id)
self.assertEqual(new_job.event_id, recipe.event_id)
self.assertEqual(new_job.recipe_id, recipe.id)
self.assertEqual(new_job.root_recipe_id, superseded_recipe.id)
self.assertEqual(new_job.root_superseded_job_id, superseded_job.id)
self.assertEqual(new_job.superseded_job_id, superseded_job.id)
self.assertEqual(new_job.batch_id, batch.id)
node_count = RecipeNode.objects.filter(recipe_id=recipe.id,
node_name=node_name,
job_id=new_job.id).count()
self.assertEqual(node_count, 1)
# Check for process_job_input message (because process_input=True)
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'process_job_input')
self.assertEqual(message.new_messages[0].job_id, new_job.id)
# Test executing message again
message.new_messages = []
result = message.execute()
self.assertTrue(result)
# Make sure a new job is not created
self.assertEqual(Job.objects.filter(recipe_id=recipe.id).count(), 1)
self.assertEqual(
RecipeNode.objects.filter(recipe_id=recipe.id,
node_name=node_name).count(), 1)
# Check for same process_job_input message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'process_job_input')
self.assertEqual(message.new_messages[0].job_id, new_job.id)
def test_execute(self):
"""Tests calling UpdateRecipeMetrics.execute() successfully"""
recipe_1 = recipe_test_utils.create_recipe()
job_1 = job_test_utils.create_job(status='FAILED')
job_2 = job_test_utils.create_job(status='CANCELED')
job_3 = job_test_utils.create_job(status='BLOCKED')
job_4 = job_test_utils.create_job(status='BLOCKED')
job_5 = job_test_utils.create_job(status='COMPLETED')
recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_1)
recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_2)
recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_3)
recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_4)
recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_5)
RecipeNode.objects.bulk_create([
recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4,
recipe_node_5
])
batch = batch_test_utils.create_batch()
recipe_2 = recipe_test_utils.create_recipe(batch=batch)
job_6 = job_test_utils.create_job(status='COMPLETED')
job_7 = job_test_utils.create_job(status='COMPLETED')
job_8 = job_test_utils.create_job(status='RUNNING')
job_9 = job_test_utils.create_job(status='QUEUED')
job_10 = job_test_utils.create_job(status='PENDING')
job_11 = job_test_utils.create_job(status='PENDING')
job_12 = job_test_utils.create_job(status='PENDING')
job_13 = job_test_utils.create_job(status='CANCELED')
job_14 = job_test_utils.create_job(status='BLOCKED')
recipe_node_6 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_6)
recipe_node_7 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_7)
recipe_node_8 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_8)
recipe_node_9 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_9)
recipe_node_10 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_10)
recipe_node_11 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_11)
recipe_node_12 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_12)
recipe_node_13 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_13)
recipe_node_14 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_14)
RecipeNode.objects.bulk_create([
recipe_node_6, recipe_node_7, recipe_node_8, recipe_node_9,
recipe_node_10, recipe_node_11, recipe_node_12, recipe_node_13,
recipe_node_14
])
# Add recipes to message
message = UpdateRecipeMetrics()
if message.can_fit_more():
message.add_recipe(recipe_1.id)
if message.can_fit_more():
message.add_recipe(recipe_2.id)
# Execute message
result = message.execute()
self.assertTrue(result)
recipe_1 = Recipe.objects.get(id=recipe_1.id)
self.assertEqual(recipe_1.jobs_total, 5)
self.assertEqual(recipe_1.jobs_pending, 0)
self.assertEqual(recipe_1.jobs_blocked, 2)
self.assertEqual(recipe_1.jobs_queued, 0)
self.assertEqual(recipe_1.jobs_running, 0)
self.assertEqual(recipe_1.jobs_failed, 1)
self.assertEqual(recipe_1.jobs_completed, 1)
self.assertEqual(recipe_1.jobs_canceled, 1)
recipe_2 = Recipe.objects.get(id=recipe_2.id)
self.assertEqual(recipe_2.jobs_total, 9)
self.assertEqual(recipe_2.jobs_pending, 3)
self.assertEqual(recipe_2.jobs_blocked, 1)
self.assertEqual(recipe_2.jobs_queued, 1)
self.assertEqual(recipe_2.jobs_running, 1)
self.assertEqual(recipe_2.jobs_failed, 0)
self.assertEqual(recipe_2.jobs_completed, 2)
self.assertEqual(recipe_2.jobs_canceled, 1)
# Make sure message is created to update batch metrics
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'update_batch_metrics')
self.assertListEqual(msg._batch_ids, [batch.id])
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipeMetrics.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
recipe_1 = Recipe.objects.get(id=recipe_1.id)
self.assertEqual(recipe_1.jobs_total, 5)
self.assertEqual(recipe_1.jobs_pending, 0)
self.assertEqual(recipe_1.jobs_blocked, 2)
self.assertEqual(recipe_1.jobs_queued, 0)
self.assertEqual(recipe_1.jobs_running, 0)
self.assertEqual(recipe_1.jobs_failed, 1)
self.assertEqual(recipe_1.jobs_completed, 1)
self.assertEqual(recipe_1.jobs_canceled, 1)
recipe_2 = Recipe.objects.get(id=recipe_2.id)
self.assertEqual(recipe_2.jobs_total, 9)
self.assertEqual(recipe_2.jobs_pending, 3)
self.assertEqual(recipe_2.jobs_blocked, 1)
self.assertEqual(recipe_2.jobs_queued, 1)
self.assertEqual(recipe_2.jobs_running, 1)
self.assertEqual(recipe_2.jobs_failed, 0)
self.assertEqual(recipe_2.jobs_completed, 2)
self.assertEqual(recipe_2.jobs_canceled, 1)
# Make sure message is created to update batch metrics
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'update_batch_metrics')
self.assertListEqual(msg._batch_ids, [batch.id])
def test_execute(self):
"""Tests calling UpdateRecipe.execute() successfully"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_a = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
recipe_b = recipe_test_utils.create_recipe()
job_c = job_test_utils.create_job(status='PENDING')
job_d = job_test_utils.create_job(status='BLOCKED')
recipe_type_e = recipe_test_utils.create_recipe_type_v6()
job_type_f = job_test_utils.create_seed_job_type()
job_type_k = job_test_utils.create_seed_job_type()
job_g = job_test_utils.create_job(status='FAILED', input=data_dict)
job_h = job_test_utils.create_job(status='PENDING')
condition_i = recipe_test_utils.create_recipe_condition(save=True)
definition = RecipeDefinition(Interface())
definition.add_job_node('node_a', job_a.job_type.name, job_a.job_type.version, job_a.job_type_rev.revision_num)
definition.add_recipe_node('node_b', recipe_b.recipe_type.name, recipe_b.recipe_type.revision_num)
definition.add_job_node('node_c', job_c.job_type.name, job_c.job_type.version, job_c.job_type_rev.revision_num)
definition.add_job_node('node_d', job_d.job_type.name, job_d.job_type.version, job_d.job_type_rev.revision_num)
definition.add_recipe_node('node_e', recipe_type_e.name, recipe_type_e.revision_num)
definition.add_job_node('node_f', job_type_f.name, job_type_f.version, job_type_f.revision_num)
definition.add_job_node('node_g', job_g.job_type.name, job_g.job_type.version, job_g.job_type_rev.revision_num)
definition.add_job_node('node_h', job_h.job_type.name, job_h.job_type.version, job_h.job_type_rev.revision_num)
definition.add_condition_node('node_i', Interface(), DataFilter()) #True
definition.add_condition_node('node_j', Interface(), DataFilter()) #True
definition.add_job_node('node_k', job_type_k.name, job_type_k.version, job_type_k.revision_num)
definition.add_dependency('node_a', 'node_c')
definition.add_dependency('node_a', 'node_e')
definition.add_dependency('node_a', 'node_g')
definition.add_dependency('node_c', 'node_d')
definition.add_dependency('node_e', 'node_f')
definition.add_dependency('node_g', 'node_h')
definition.add_dependency('node_a', 'node_i')
definition.add_dependency('node_a', 'node_j')
definition.add_dependency('node_i', 'node_k')
definition.add_dependency('node_j', 'node_k')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', job=job_a, save=False)
node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_b', sub_recipe=recipe_b,
save=False)
node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_c', job=job_c, save=False)
node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_d', job=job_d, save=False)
node_g = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_g', job=job_g, save=False)
node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_h', job=job_h, save=False)
node_i = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_i', condition=condition_i,
save=False)
RecipeNode.objects.bulk_create([node_a, node_b, node_c, node_d, node_g, node_h, node_i])
forced_nodes = ForcedNodes()
forced_nodes_e = ForcedNodes()
forced_nodes_e.set_all_nodes()
forced_nodes.add_subrecipe('node_e', forced_nodes_e)
# Create and execute message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
result = message.execute()
self.assertTrue(result)
self.assertEqual(len(message.new_messages), 8)
# Check messages
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipe.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Make sure the same messages are returned
self.assertEqual(len(message.new_messages), 8)
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
def test_has_completed_false(self):
"""Tests calling Recipe.has_completed() when an entire recipe has not completed"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.is_completed = True
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.is_completed = False
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
self.assertFalse(recipe_instance.has_completed())
def test_get_jobs_to_update(self):
"""Tests calling Recipe.get_jobs_to_update()"""
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('B', 'E')
definition.add_dependency('B', 'G')
definition.add_dependency('C', 'D')
definition.add_dependency('E', 'F')
definition.add_dependency('G', 'H')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='CANCELED',
num_exes=0,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='PENDING',
num_exes=0,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='BLOCKED',
num_exes=0,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='PENDING',
num_exes=0,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='PENDING',
num_exes=0,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.jobs_completed = 3
recipe_b.jobs_running = 2
recipe_b.jobs_total = 5
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.jobs_completed = 2
recipe_g.jobs_failed = 1
recipe_g.jobs_total = 3
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
results = recipe_instance.get_jobs_to_update()
self.assertSetEqual(set(results['BLOCKED']), {job_d.id, job_h.id})
self.assertSetEqual(set(results['PENDING']), {job_e.id})
def test_convert_recipe_to_v6_json(self):
"""Tests calling convert_recipe_to_v6_json() successfully"""
job_type_1 = job_test_utils.create_job_type()
job_type_2 = job_test_utils.create_job_type()
job_type_3 = job_test_utils.create_job_type()
job_type_4 = job_test_utils.create_job_type()
recipe_type_1 = recipe_test_utils.create_recipe_type()
interface = Interface()
interface.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_1', 'object'))
definition = RecipeDefinition(interface)
definition.add_job_node('A', job_type_1.name, job_type_1.version,
job_type_1.revision_num)
definition.add_job_node('B', job_type_2.name, job_type_2.version,
job_type_2.revision_num)
definition.add_job_node('C', job_type_3.name, job_type_3.version,
job_type_3.revision_num)
definition.add_recipe_node('D', recipe_type_1.name,
recipe_type_1.revision_num)
definition.add_job_node('E', job_type_4.name, job_type_4.version,
job_type_4.revision_num)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_1')
definition.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
recipe = recipe_test_utils.create_recipe()
job_a = job_test_utils.create_job(job_type=job_type_1,
status='COMPLETED',
save=False)
job_b = job_test_utils.create_job(job_type=job_type_2,
status='RUNNING',
save=False)
job_c = job_test_utils.create_job(job_type=job_type_3,
status='COMPLETED',
save=False)
job_e = job_test_utils.create_job(job_type=job_type_4,
status='PENDING',
num_exes=0,
save=False)
Job.objects.bulk_create([job_a, job_b, job_c, job_e])
recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='B',
job=job_b,
save=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_nodes = [
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e
]
recipe_instance = RecipeInstance(definition, recipe_nodes)
json = convert_recipe_to_v6_json(recipe_instance)
RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()),
{'A', 'B', 'C', 'D', 'E'})
def test_execute_recipe_superseded(self):
"""Tests calling CreateJobs.execute() successfully for jobs within a recipe that supersedes another recipe"""
from batch.test import utils as batch_test_utils
from recipe.models import RecipeNode
from recipe.test import utils as recipe_test_utils
batch = batch_test_utils.create_batch()
event = trigger_test_utils.create_trigger_event()
job_type_1 = job_test_utils.create_seed_job_type()
job_type_2 = job_test_utils.create_seed_job_type()
superseded_recipe = recipe_test_utils.create_recipe()
superseded_job_1 = job_test_utils.create_job(job_type=job_type_1,
is_superseded=True)
superseded_job_2 = job_test_utils.create_job(job_type=job_type_2,
is_superseded=True)
recipe_test_utils.create_recipe_node(recipe=superseded_recipe,
node_name='node_1',
job=superseded_job_1,
save=True)
recipe_test_utils.create_recipe_node(recipe=superseded_recipe,
node_name='node_2',
job=superseded_job_2,
save=True)
recipe = recipe_test_utils.create_recipe(
recipe_type=superseded_recipe.recipe_type,
superseded_recipe=superseded_recipe,
event=event,
batch=batch)
recipe_jobs = [
RecipeJob(job_type_1.name, job_type_1.version,
job_type_1.revision_num, 'node_1', False),
RecipeJob(job_type_2.name, job_type_2.version,
job_type_2.revision_num, 'node_2', True)
]
# Create and execute message
message = create_jobs_messages_for_recipe(recipe, recipe_jobs)[0]
result = message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(recipe_id=recipe.id,
event_id=recipe.event_id).count(), 2)
recipe_nodes = RecipeNode.objects.select_related('job').filter(
recipe_id=recipe.id)
self.assertEqual(len(recipe_nodes), 2)
for recipe_node in recipe_nodes:
if recipe_node.node_name == 'node_1':
job_1 = recipe_node.job
self.assertEqual(job_1.job_type_id, job_type_1.id)
self.assertEqual(job_1.event_id, event.id)
self.assertEqual(job_1.batch_id, batch.id)
self.assertEqual(job_1.recipe_id, recipe.id)
self.assertEqual(job_1.root_recipe_id, superseded_recipe.id)
self.assertEqual(job_1.superseded_job_id, superseded_job_1.id)
self.assertEqual(job_1.root_superseded_job_id,
superseded_job_1.id)
elif recipe_node.node_name == 'node_2':
job_2 = recipe_node.job
self.assertEqual(job_2.job_type_id, job_type_2.id)
self.assertEqual(job_2.event_id, event.id)
self.assertEqual(job_2.batch_id, batch.id)
self.assertEqual(job_2.recipe_id, recipe.id)
self.assertEqual(job_2.root_recipe_id, superseded_recipe.id)
self.assertEqual(job_2.superseded_job_id, superseded_job_2.id)
self.assertEqual(job_2.root_superseded_job_id,
superseded_job_2.id)
else:
self.fail('%s is the wrong node name' % recipe_node.node_name)
# Should be two messages, one for processing job input and one for updating metrics for the recipe
self.assertEqual(len(message.new_messages), 2)
process_job_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message to process job input for new job 2
self.assertEqual(process_job_input_msg.job_id, job_2.id)
# Check message to update recipe metrics for the recipe containing the new jobs
self.assertListEqual(update_metrics_msg._recipe_ids, [recipe.id])
# Test executing message again
message_json_dict = message.to_json()
message = CreateJobs.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(recipe_id=recipe.id,
event_id=recipe.event_id).count(), 2)
recipe_nodes = RecipeNode.objects.select_related('job').filter(
recipe_id=recipe.id)
self.assertEqual(len(recipe_nodes), 2)
for recipe_node in recipe_nodes:
if recipe_node.node_name == 'node_1':
job_1 = recipe_node.job
self.assertEqual(job_1.job_type_id, job_type_1.id)
self.assertEqual(job_1.event_id, event.id)
self.assertEqual(job_1.batch_id, batch.id)
self.assertEqual(job_1.recipe_id, recipe.id)
self.assertEqual(job_1.root_recipe_id, superseded_recipe.id)
self.assertEqual(job_1.superseded_job_id, superseded_job_1.id)
self.assertEqual(job_1.root_superseded_job_id,
superseded_job_1.id)
elif recipe_node.node_name == 'node_2':
job_2 = recipe_node.job
self.assertEqual(job_2.job_type_id, job_type_2.id)
self.assertEqual(job_2.event_id, event.id)
self.assertEqual(job_2.batch_id, batch.id)
self.assertEqual(job_2.recipe_id, recipe.id)
self.assertEqual(job_2.root_recipe_id, superseded_recipe.id)
self.assertEqual(job_2.superseded_job_id, superseded_job_2.id)
self.assertEqual(job_2.root_superseded_job_id,
superseded_job_2.id)
else:
self.fail('%s is the wrong node name' % recipe_node.node_name)
# Should be two messages, one for processing job input and one for updating metrics for the recipe
self.assertEqual(len(message.new_messages), 2)
process_job_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message to process job input for new job 2
self.assertEqual(process_job_input_msg.job_id, job_2.id)
# Check message to update recipe metrics for the recipe containing the new jobs
self.assertListEqual(update_metrics_msg._recipe_ids, [recipe.id])
def test_get_nodes_to_process_input(self):
"""Tests calling Recipe.get_nodes_to_process_input()"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
# Create recipe
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=False) #always True
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition.add_condition_node('B', cond_interface_1, df1) #True
definition.add_condition_node('C', cond_interface_1, df1) #True
definition.add_condition_node('D', cond_interface_1, df2) #False
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_recipe_node('H', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_dependency('A', 'D')
definition.add_dependency('A', 'E')
definition.add_dependency('B', 'E')
definition.add_dependency('B', 'F')
definition.add_dependency('C', 'F')
definition.add_dependency('D', 'G')
definition.add_dependency('E', 'G')
definition.add_dependency('E', 'H')
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
# Nodes A, B, and D already exist
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
input=data_dict,
output=data_dict,
save=True)
condition_b = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=True, save=False)
condition_d = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=False)
RecipeCondition.objects.bulk_create([condition_b, condition_d])
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', condition=condition_b, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', condition=condition_d, save=False)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_d])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
nodes_to_process = recipe_instance.get_nodes_to_process_input()
self.assertSetEqual(set(nodes_to_process.keys()), {'C', 'E'})
def test_execute_with_sub_recipes(self):
"""Tests calling UpdateRecipeMetrics.execute() successfully with sub-recipes"""
recipe_1 = recipe_test_utils.create_recipe()
batch = batch_test_utils.create_batch()
recipe_2 = recipe_test_utils.create_recipe(batch=batch)
# Recipe 1 jobs
job_1 = job_test_utils.create_job(status='FAILED', save=False)
job_2 = job_test_utils.create_job(status='CANCELED', save=False)
job_3 = job_test_utils.create_job(status='BLOCKED', save=False)
job_4 = job_test_utils.create_job(status='BLOCKED', save=False)
job_5 = job_test_utils.create_job(status='COMPLETED', save=False)
# Recipe 2 jobs
job_6 = job_test_utils.create_job(status='COMPLETED', save=False)
job_7 = job_test_utils.create_job(status='COMPLETED', save=False)
job_8 = job_test_utils.create_job(status='RUNNING', save=False)
job_9 = job_test_utils.create_job(status='QUEUED', save=False)
job_10 = job_test_utils.create_job(status='PENDING', save=False)
job_11 = job_test_utils.create_job(status='PENDING', save=False)
job_12 = job_test_utils.create_job(status='PENDING', save=False)
job_13 = job_test_utils.create_job(status='CANCELED', save=False)
job_14 = job_test_utils.create_job(status='BLOCKED', save=False)
Job.objects.bulk_create([
job_1, job_2, job_3, job_4, job_5, job_6, job_7, job_8, job_9,
job_10, job_11, job_12, job_13, job_14
])
# Recipe 1 sub-recipes
sub_recipe_1 = recipe_test_utils.create_recipe(save=False)
sub_recipe_1.jobs_total = 26
sub_recipe_1.jobs_pending = 3
sub_recipe_1.jobs_blocked = 4
sub_recipe_1.jobs_queued = 5
sub_recipe_1.jobs_running = 1
sub_recipe_1.jobs_failed = 2
sub_recipe_1.jobs_completed = 3
sub_recipe_1.jobs_canceled = 8
sub_recipe_1.is_completed = False
sub_recipe_2 = recipe_test_utils.create_recipe(save=False)
sub_recipe_2.jobs_total = 30
sub_recipe_2.jobs_completed = 30
sub_recipe_2.is_completed = True
# Recipe 2 sub-recipes
sub_recipe_3 = recipe_test_utils.create_recipe(save=False)
sub_recipe_3.jobs_total = 21
sub_recipe_3.jobs_pending = 2
sub_recipe_3.jobs_blocked = 5
sub_recipe_3.jobs_queued = 0
sub_recipe_3.jobs_running = 3
sub_recipe_3.jobs_failed = 2
sub_recipe_3.jobs_completed = 8
sub_recipe_3.jobs_canceled = 1
sub_recipe_3.is_completed = False
sub_recipe_4 = recipe_test_utils.create_recipe(save=False)
sub_recipe_4.jobs_total = 7
sub_recipe_4.jobs_completed = 7
sub_recipe_4.is_completed = True
sub_recipe_5 = recipe_test_utils.create_recipe(save=False)
sub_recipe_5.jobs_total = 12
sub_recipe_5.jobs_completed = 12
sub_recipe_5.is_completed = True
Recipe.objects.bulk_create([
sub_recipe_1, sub_recipe_2, sub_recipe_3, sub_recipe_4,
sub_recipe_5
])
# Recipe 1 nodes
recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_1)
recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_2)
recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_3)
recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_4)
recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe_1,
job=job_5)
recipe_node_6 = recipe_test_utils.create_recipe_node(
recipe=recipe_1, sub_recipe=sub_recipe_1)
recipe_node_7 = recipe_test_utils.create_recipe_node(
recipe=recipe_1, sub_recipe=sub_recipe_2)
# Recipe 2 nodes
recipe_node_8 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_6)
recipe_node_9 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_7)
recipe_node_10 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_8)
recipe_node_11 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_9)
recipe_node_12 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_10)
recipe_node_13 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_11)
recipe_node_14 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_12)
recipe_node_15 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_13)
recipe_node_16 = recipe_test_utils.create_recipe_node(recipe=recipe_2,
job=job_14)
recipe_node_17 = recipe_test_utils.create_recipe_node(
recipe=recipe_2, sub_recipe=sub_recipe_3)
recipe_node_18 = recipe_test_utils.create_recipe_node(
recipe=recipe_2, sub_recipe=sub_recipe_4)
recipe_node_19 = recipe_test_utils.create_recipe_node(
recipe=recipe_2, sub_recipe=sub_recipe_5)
RecipeNode.objects.bulk_create([
recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4,
recipe_node_5, recipe_node_6, recipe_node_7, recipe_node_8,
recipe_node_9, recipe_node_10, recipe_node_11, recipe_node_12,
recipe_node_13, recipe_node_14, recipe_node_15, recipe_node_16,
recipe_node_17, recipe_node_18, recipe_node_19
])
# Add recipes to message
message = UpdateRecipeMetrics()
if message.can_fit_more():
message.add_recipe(recipe_1.id)
if message.can_fit_more():
message.add_recipe(recipe_2.id)
# Execute message
result = message.execute()
self.assertTrue(result)
recipe_1 = Recipe.objects.get(id=recipe_1.id)
self.assertEqual(recipe_1.jobs_total, 61)
self.assertEqual(recipe_1.jobs_pending, 3)
self.assertEqual(recipe_1.jobs_blocked, 6)
self.assertEqual(recipe_1.jobs_queued, 5)
self.assertEqual(recipe_1.jobs_running, 1)
self.assertEqual(recipe_1.jobs_failed, 3)
self.assertEqual(recipe_1.jobs_completed, 34)
self.assertEqual(recipe_1.jobs_canceled, 9)
self.assertEqual(recipe_1.sub_recipes_total, 2)
self.assertEqual(recipe_1.sub_recipes_completed, 1)
recipe_2 = Recipe.objects.get(id=recipe_2.id)
self.assertEqual(recipe_2.jobs_total, 49)
self.assertEqual(recipe_2.jobs_pending, 5)
self.assertEqual(recipe_2.jobs_blocked, 6)
self.assertEqual(recipe_2.jobs_queued, 1)
self.assertEqual(recipe_2.jobs_running, 4)
self.assertEqual(recipe_2.jobs_failed, 2)
self.assertEqual(recipe_2.jobs_completed, 29)
self.assertEqual(recipe_2.jobs_canceled, 2)
self.assertEqual(recipe_2.sub_recipes_total, 3)
self.assertEqual(recipe_2.sub_recipes_completed, 2)
# Make sure message is created to update batch metrics
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'update_batch_metrics')
self.assertListEqual(msg._batch_ids, [batch.id])
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipeMetrics.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
recipe_1 = Recipe.objects.get(id=recipe_1.id)
self.assertEqual(recipe_1.jobs_total, 61)
self.assertEqual(recipe_1.jobs_pending, 3)
self.assertEqual(recipe_1.jobs_blocked, 6)
self.assertEqual(recipe_1.jobs_queued, 5)
self.assertEqual(recipe_1.jobs_running, 1)
self.assertEqual(recipe_1.jobs_failed, 3)
self.assertEqual(recipe_1.jobs_completed, 34)
self.assertEqual(recipe_1.jobs_canceled, 9)
self.assertEqual(recipe_1.sub_recipes_total, 2)
self.assertEqual(recipe_1.sub_recipes_completed, 1)
recipe_2 = Recipe.objects.get(id=recipe_2.id)
self.assertEqual(recipe_2.jobs_total, 49)
self.assertEqual(recipe_2.jobs_pending, 5)
self.assertEqual(recipe_2.jobs_blocked, 6)
self.assertEqual(recipe_2.jobs_queued, 1)
self.assertEqual(recipe_2.jobs_running, 4)
self.assertEqual(recipe_2.jobs_failed, 2)
self.assertEqual(recipe_2.jobs_completed, 29)
self.assertEqual(recipe_2.jobs_canceled, 2)
self.assertEqual(recipe_2.sub_recipes_total, 3)
self.assertEqual(recipe_2.sub_recipes_completed, 2)
# Make sure message is created to update batch metrics
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'update_batch_metrics')
self.assertListEqual(msg._batch_ids, [batch.id])
def test_get_original_leaf_nodes(self):
"""Tests calling Recipe.get_original_leaf_nodes()"""
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
save=False,
is_superseded=True)
job_c = job_test_utils.create_job(job_type=job_type,
status='CANCELED',
num_exes=0,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='PENDING',
num_exes=0,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='BLOCKED',
num_exes=0,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='PENDING',
num_exes=0,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='PENDING',
num_exes=0,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.jobs_completed = 3
recipe_b.jobs_running = 2
recipe_b.jobs_total = 5
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.jobs_completed = 2
recipe_g.jobs_failed = 1
recipe_g.jobs_total = 3
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
results = recipe_instance.get_original_leaf_nodes()
self.assertEqual(len(results.values()), 4)
leaf_jobs = [
node.job.id for node in results.values()
if node.node_type == JobNodeDefinition.NODE_TYPE
]
leaf_recipes = [
node.recipe.id for node in results.values()
if node.node_type == RecipeNodeDefinition.NODE_TYPE
]
self.assertItemsEqual(
leaf_jobs,
[recipe_node_e.job.id, recipe_node_f.job.id, recipe_node_h.job.id])
self.assertItemsEqual(leaf_recipes, [recipe_node_b.sub_recipe.id])
def test_execute_with_top_level_recipe(self):
"""Tests calling UpdateRecipeMetrics.execute() successfully where a message needs to be sent to update a
top-level recipe
"""
batch = batch_test_utils.create_batch()
top_recipe = recipe_test_utils.create_recipe(batch=batch)
recipe = recipe_test_utils.create_recipe(batch=batch)
recipe.recipe = top_recipe
recipe.save()
recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=top_recipe,
sub_recipe=recipe)
# Recipe jobs
job_1 = job_test_utils.create_job(status='FAILED', save=False)
job_2 = job_test_utils.create_job(status='CANCELED', save=False)
job_3 = job_test_utils.create_job(status='BLOCKED', save=False)
job_4 = job_test_utils.create_job(status='BLOCKED', save=False)
job_5 = job_test_utils.create_job(status='COMPLETED', save=False)
Job.objects.bulk_create([job_1, job_2, job_3, job_4, job_5])
# Recipe nodes
recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_1)
recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_2)
recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_3)
recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_4)
recipe_node_6 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_5)
RecipeNode.objects.bulk_create([
recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4,
recipe_node_5, recipe_node_6
])
# Add recipes to message
message = UpdateRecipeMetrics()
if message.can_fit_more():
message.add_recipe(recipe.id)
# Execute message
result = message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(recipe.jobs_total, 5)
self.assertEqual(recipe.jobs_pending, 0)
self.assertEqual(recipe.jobs_blocked, 2)
self.assertEqual(recipe.jobs_queued, 0)
self.assertEqual(recipe.jobs_running, 0)
self.assertEqual(recipe.jobs_failed, 1)
self.assertEqual(recipe.jobs_completed, 1)
self.assertEqual(recipe.jobs_canceled, 1)
self.assertEqual(recipe.sub_recipes_total, 0)
self.assertEqual(recipe.sub_recipes_completed, 0)
# Make sure message is created to update top-level recipe metrics
# There should be no message to update batch metrics since we did not update a top-level recipe
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'update_recipe_metrics')
self.assertListEqual(msg._recipe_ids, [top_recipe.id])
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipeMetrics.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(recipe.jobs_total, 5)
self.assertEqual(recipe.jobs_pending, 0)
self.assertEqual(recipe.jobs_blocked, 2)
self.assertEqual(recipe.jobs_queued, 0)
self.assertEqual(recipe.jobs_running, 0)
self.assertEqual(recipe.jobs_failed, 1)
self.assertEqual(recipe.jobs_completed, 1)
self.assertEqual(recipe.jobs_canceled, 1)
self.assertEqual(recipe.sub_recipes_total, 0)
self.assertEqual(recipe.sub_recipes_completed, 0)
# Make sure message is created to update top-level recipe metrics
# There should be no message to update batch metrics since we did not update a top-level recipe
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'update_recipe_metrics')
self.assertListEqual(msg._recipe_ids, [top_recipe.id])
def test_has_completed_true(self):
"""Tests calling Recipe.has_completed() when an entire recipe has completed"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_condition_node('I', cond_interface_1, df2) #False
definition.add_job_node('J', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
definition.add_dependency('A', 'I')
definition.add_dependency('I', 'J')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
condition_i = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=True)
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.is_completed = True
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.is_completed = True
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_i = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='I', condition=condition_i, save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h,
recipe_node_i
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
self.assertTrue(recipe_instance.has_completed())
def test_execute_subrecipes_superseded(self):
"""Tests calling CreateRecipes.execute() successfully when creating sub-recipes that supersede other sub-recipes
"""
# Creates definitions for sub-recipe A and sub-recipe B
batch = batch_test_utils.create_batch()
event = trigger_test_utils.create_trigger_event()
top_recipe_type = recipe_test_utils.create_recipe_type()
job_type_a_1 = job_test_utils.create_seed_job_type()
job_type_a_2 = job_test_utils.create_seed_job_type()
sub_definition_a = RecipeDefinition(Interface())
sub_definition_a.add_job_node('node_1', job_type_a_1.name,
job_type_a_1.version,
job_type_a_1.revision_num)
sub_definition_a.add_job_node('node_2', job_type_a_2.name,
job_type_a_2.version,
job_type_a_2.revision_num)
sub_definition_a.add_dependency('node_1', 'node_2')
sub_definition_a_dict = convert_recipe_definition_to_v6_json(
sub_definition_a).get_dict()
recipe_type_a = recipe_test_utils.create_recipe_type(
definition=sub_definition_a_dict)
job_type_b_x = job_test_utils.create_seed_job_type()
job_type_b_y = job_test_utils.create_seed_job_type()
recipe_type_b_z = recipe_test_utils.create_recipe_type()
sub_definition_b = RecipeDefinition(Interface())
sub_definition_b.add_job_node('node_x', job_type_b_x.name,
job_type_b_x.version,
job_type_b_x.revision_num)
sub_definition_b.add_job_node('node_y', job_type_b_y.name,
job_type_b_y.version,
job_type_b_y.revision_num)
sub_definition_b.add_recipe_node('node_z', recipe_type_b_z.name,
recipe_type_b_z.revision_num)
sub_definition_b.add_dependency('node_x', 'node_z')
sub_definition_b.add_dependency('node_y', 'node_z')
sub_definition_b_dict = convert_recipe_definition_to_v6_json(
sub_definition_b).get_dict()
recipe_type_b = recipe_test_utils.create_recipe_type(
definition=sub_definition_b_dict)
# Create previous recipe containing sub-recipe A and B in order to be superseded
prev_recipe_a = recipe_test_utils.create_recipe(
recipe_type=recipe_type_a, save=False)
prev_job_a_1 = job_test_utils.create_job(job_type=job_type_a_1,
save=False)
prev_job_a_2 = job_test_utils.create_job(job_type=job_type_a_2,
save=False)
prev_recipe_b = recipe_test_utils.create_recipe(
recipe_type=recipe_type_b, save=False)
prev_job_b_x = job_test_utils.create_job(job_type=job_type_b_x,
save=False)
prev_job_b_y = job_test_utils.create_job(job_type=job_type_b_y,
save=False)
prev_recipe_b_z = recipe_test_utils.create_recipe(
recipe_type=recipe_type_b_z, save=False)
prev_top_recipe = recipe_test_utils.create_recipe(
recipe_type=top_recipe_type, save=False)
new_top_recipe = recipe_test_utils.create_recipe(
recipe_type=top_recipe_type, save=False)
Job.objects.bulk_create(
[prev_job_a_1, prev_job_a_2, prev_job_b_x, prev_job_b_y])
Recipe.objects.bulk_create([
prev_recipe_a, prev_recipe_b, prev_recipe_b_z, prev_top_recipe,
new_top_recipe
])
recipe_node_a = recipe_test_utils.create_recipe_node(
recipe=prev_top_recipe,
sub_recipe=prev_recipe_a,
node_name='node_a',
save=False)
recipe_node_a_1 = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_a,
job=prev_job_a_1,
node_name='node_1',
save=False)
recipe_node_a_2 = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_a,
job=prev_job_a_2,
node_name='node_2',
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=prev_top_recipe,
sub_recipe=prev_recipe_b,
node_name='node_b',
save=False)
recipe_node_b_x = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_b,
job=prev_job_b_x,
node_name='node_x',
save=False)
recipe_node_b_y = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_b,
job=prev_job_b_y,
node_name='node_y',
save=False)
recipe_node_b_z = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_b,
sub_recipe=prev_recipe_b_z,
node_name='node_z',
save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_a_1, recipe_node_a_2, recipe_node_b,
recipe_node_b_x, recipe_node_b_y, recipe_node_b_z
])
# Create message to create sub-recipes A and B for new_top_recipe which supersedes prev_top_recipe
sub_recipes = [
SubRecipe(recipe_type_a.name, recipe_type_a.revision_num, 'node_a',
True),
SubRecipe(recipe_type_b.name, recipe_type_b.revision_num, 'node_b',
False)
]
forced_nodes = ForcedNodes()
sub_forced_nodes_b = ForcedNodes()
sub_forced_nodes_y = ForcedNodes()
sub_forced_nodes_b.add_subrecipe('node_y', sub_forced_nodes_y)
forced_nodes.add_subrecipe('node_b', sub_forced_nodes_b)
message = create_subrecipes_messages(
new_top_recipe.id,
new_top_recipe.root_superseded_recipe_id,
sub_recipes,
event.id,
superseded_recipe_id=prev_top_recipe.id,
forced_nodes=forced_nodes,
batch_id=batch.id)[0]
# Execute message
result = message.execute()
self.assertTrue(result)
# Check for new sub-recipes
qry = RecipeNode.objects.select_related('sub_recipe')
recipe_nodes = qry.filter(
recipe_id=new_top_recipe.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_a')
self.assertEqual(recipe_nodes[1].node_name, 'node_b')
sub_recipe_a = recipe_nodes[0].sub_recipe
sub_recipe_b = recipe_nodes[1].sub_recipe
self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id)
self.assertEqual(sub_recipe_a.superseded_recipe_id, prev_recipe_a.id)
self.assertEqual(sub_recipe_a.root_superseded_recipe_id,
prev_recipe_a.id)
self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id)
self.assertEqual(sub_recipe_b.superseded_recipe_id, prev_recipe_b.id)
self.assertEqual(sub_recipe_b.root_superseded_recipe_id,
prev_recipe_b.id)
# Check for sub-recipes to contain correct copied nodes
# Nodes 1 and 2 in sub-recipe A should be copied
recipe_nodes = RecipeNode.objects.select_related('job').filter(
recipe_id=sub_recipe_a.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_1')
self.assertFalse(recipe_nodes[0].is_original)
self.assertEqual(recipe_nodes[0].job_id, prev_job_a_1.id)
self.assertEqual(recipe_nodes[1].node_name, 'node_2')
self.assertFalse(recipe_nodes[1].is_original)
self.assertEqual(recipe_nodes[1].job_id, prev_job_a_2.id)
# Node X in sub-recipe B should be copied
recipe_nodes = RecipeNode.objects.select_related('sub_recipe').filter(
recipe_id=sub_recipe_b.id)
self.assertEqual(len(recipe_nodes), 1)
self.assertEqual(recipe_nodes[0].node_name, 'node_x')
self.assertFalse(recipe_nodes[0].is_original)
# Should be four messages, two for superseding recipe nodes, one for processing recipe input, and one for
# updating metrics for the recipe containing the new sub-recipes
self.assertEqual(len(message.new_messages), 4)
supersede_recipe_a_msg = None
supersede_recipe_b_msg = None
process_recipe_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'supersede_recipe_nodes':
if msg._recipe_ids[0] == prev_recipe_a.id:
supersede_recipe_a_msg = msg
if msg._recipe_ids[0] == prev_recipe_b.id:
supersede_recipe_b_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(supersede_recipe_a_msg)
self.assertIsNotNone(supersede_recipe_b_msg)
self.assertIsNotNone(process_recipe_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message for superseding previous sub-recipe A
self.assertFalse(supersede_recipe_a_msg.supersede_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_jobs, set())
self.assertSetEqual(supersede_recipe_a_msg.supersede_subrecipes, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_a_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_recursive, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_recursive, set())
# Check message for superseding previous sub-recipe B
self.assertFalse(supersede_recipe_b_msg.supersede_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_jobs, {'node_y'})
self.assertSetEqual(supersede_recipe_b_msg.supersede_subrecipes,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_b_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_recursive,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_recursive, set())
# Check message to process recipe input for new sub-recipe A
self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id)
# Check message to update recipe metrics for the recipe containing the new sub-recipes
self.assertListEqual(update_metrics_msg._recipe_ids,
[new_top_recipe.id])
# Test executing message again
message_json_dict = message.to_json()
message = CreateRecipes.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Check sub-recipes to make sure we didn't create them again a second time
qry = RecipeNode.objects.select_related('sub_recipe')
recipe_nodes = qry.filter(
recipe_id=new_top_recipe.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_a')
self.assertEqual(recipe_nodes[1].node_name, 'node_b')
sub_recipe_a = recipe_nodes[0].sub_recipe
sub_recipe_b = recipe_nodes[1].sub_recipe
self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id)
self.assertEqual(sub_recipe_a.superseded_recipe_id, prev_recipe_a.id)
self.assertEqual(sub_recipe_a.root_superseded_recipe_id,
prev_recipe_a.id)
self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id)
self.assertEqual(sub_recipe_b.superseded_recipe_id, prev_recipe_b.id)
self.assertEqual(sub_recipe_b.root_superseded_recipe_id,
prev_recipe_b.id)
# Check for sub-recipes to contain correct copied nodes
# Nodes 1 and 2 in sub-recipe A should be copied
recipe_nodes = RecipeNode.objects.select_related('job').filter(
recipe_id=sub_recipe_a.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_1')
self.assertFalse(recipe_nodes[0].is_original)
self.assertEqual(recipe_nodes[0].job_id, prev_job_a_1.id)
self.assertEqual(recipe_nodes[1].node_name, 'node_2')
self.assertFalse(recipe_nodes[1].is_original)
self.assertEqual(recipe_nodes[1].job_id, prev_job_a_2.id)
# Node X in sub-recipe B should be copied
recipe_nodes = RecipeNode.objects.select_related('sub_recipe').filter(
recipe_id=sub_recipe_b.id)
self.assertEqual(len(recipe_nodes), 1)
self.assertEqual(recipe_nodes[0].node_name, 'node_x')
self.assertFalse(recipe_nodes[0].is_original)
# Check messages again
# Should be four messages, two for superseding recipe nodes, one for processing recipe input, and one for
# updating metrics for the recipe containing the new sub-recipes
self.assertEqual(len(message.new_messages), 4)
supersede_recipe_a_msg = None
supersede_recipe_b_msg = None
process_recipe_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'supersede_recipe_nodes':
if msg._recipe_ids[0] == prev_recipe_a.id:
supersede_recipe_a_msg = msg
if msg._recipe_ids[0] == prev_recipe_b.id:
supersede_recipe_b_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(supersede_recipe_a_msg)
self.assertIsNotNone(supersede_recipe_b_msg)
self.assertIsNotNone(process_recipe_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message for superseding previous sub-recipe A
self.assertFalse(supersede_recipe_a_msg.supersede_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_jobs, set())
self.assertSetEqual(supersede_recipe_a_msg.supersede_subrecipes, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_a_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_recursive, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_recursive, set())
# Check message for superseding previous sub-recipe B
self.assertFalse(supersede_recipe_b_msg.supersede_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_jobs, {'node_y'})
self.assertSetEqual(supersede_recipe_b_msg.supersede_subrecipes,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_b_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_recursive,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_recursive, set())
# Check message to process recipe input for new sub-recipe A
self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id)
# Check message to update recipe metrics for the recipe containing the new sub-recipes
self.assertListEqual(update_metrics_msg._recipe_ids,
[new_top_recipe.id])
