def test_json_forced_nodes(self):
"""Tests converting an UpdateRecipe message to and from JSON when forced nodes are provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_completed = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('job_completed', job_completed.job_type.name, job_completed.job_type.version,
job_completed.job_type_rev.revision_num)
definition.add_recipe_node('the_sub_recipe', sub_recipe_type.name, sub_recipe_type.revision_num)
definition.add_dependency('job_completed', 'the_sub_recipe')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_completed', job=job_completed)
forced_nodes = ForcedNodes()
sub_forced_nodes = ForcedNodes()
sub_forced_nodes.set_all_nodes()
forced_nodes.add_subrecipe('the_sub_recipe', sub_forced_nodes)
# Create message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to create sub-recipe
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'create_recipes')
self.assertEqual(msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(msg.recipe_id, recipe.id)
self.assertEqual(msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(msg.superseded_recipe_id)
sub = SubRecipe(sub_recipe_type.name, sub_recipe_type.revision_num, 'the_sub_recipe', True)
self.assertListEqual(msg.sub_recipes, [sub])
def update_recipe(root_recipe_id):
"""Mimics effect of update recipe messages for unit testing """
recipe = Recipe.objects.get_recipe_instance_from_root(root_recipe_id)
recipe_model = recipe.recipe_model
when = timezone.now()
jobs_to_update = recipe.get_jobs_to_update()
blocked_job_ids = jobs_to_update['BLOCKED']
pending_job_ids = jobs_to_update['PENDING']
nodes_to_create = recipe.get_nodes_to_create()
nodes_to_process_input = recipe.get_nodes_to_process_input()
if not recipe_model.is_completed and recipe.has_completed():
Recipe.objects.complete_recipes([recipe_model.id], when)
# Create new messages for changing job statuses
if len(blocked_job_ids):
update_jobs_status(blocked_job_ids, when, status='BLOCKED')
if len(pending_job_ids):
update_jobs_status(pending_job_ids, when, status='PENDING')
# Create recipe nodes
conditions = []
recipe_jobs = []
subrecipes = []
for node_name, node_def in nodes_to_create.items():
process_input = False
if node_name in nodes_to_process_input:
process_input = True
del nodes_to_process_input[node_name]
if node_def.node_type == ConditionNodeDefinition.NODE_TYPE:
condition = Condition(node_name, process_input)
conditions.append(condition)
elif node_def.node_type == JobNodeDefinition.NODE_TYPE:
job = RecipeJob(node_def.job_type_name, node_def.job_type_version, node_def.revision_num, node_name,
process_input)
recipe_jobs.append(job)
elif node_def.node_type == RecipeNodeDefinition.NODE_TYPE:
subrecipe = SubRecipe(node_def.recipe_type_name, node_def.revision_num, node_name, process_input)
subrecipes.append(subrecipe)
if len(conditions):
create_conditions(recipe_model, conditions)
if len(recipe_jobs):
create_jobs_for_recipe(recipe_model, recipe_jobs)
if len(subrecipes):
create_subrecipes(recipe_model, subrecipes)
# Create new messages for processing recipe node input
process_condition_ids = []
process_job_ids = []
process_recipe_ids = []
for node_name, node in nodes_to_process_input.items():
if node.node_type == ConditionNodeDefinition.NODE_TYPE:
process_condition_ids.append(node.condition.id)
elif node.node_type == JobNodeDefinition.NODE_TYPE:
process_job_ids.append(node.job.id)
elif node.node_type == RecipeNodeDefinition.NODE_TYPE:
process_recipe_ids.append(node.recipe.id)
if len(process_condition_ids):
process_conditions(process_condition_ids)
if len(process_job_ids):
process_job_inputs(process_job_ids)
if len(process_recipe_ids):
process_recipe_inputs(process_recipe_ids)
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
recipe = Recipe.objects.get_recipe_instance_from_root(
self.root_recipe_id)
recipe_model = recipe.recipe_model
when = now()
jobs_to_update = recipe.get_jobs_to_update()
blocked_job_ids = jobs_to_update['BLOCKED']
pending_job_ids = jobs_to_update['PENDING']
nodes_to_create = recipe.get_nodes_to_create()
nodes_to_process_input = recipe.get_nodes_to_process_input()
if not recipe_model.is_completed and recipe.has_completed():
Recipe.objects.complete_recipes([recipe_model.id], when)
# Create new messages for changing job statuses
if len(blocked_job_ids):
logger.info('Found %d job(s) that should transition to BLOCKED',
len(blocked_job_ids))
self.new_messages.extend(
create_blocked_jobs_messages(blocked_job_ids, when))
if len(pending_job_ids):
logger.info('Found %d job(s) that should transition to PENDING',
len(pending_job_ids))
self.new_messages.extend(
create_pending_jobs_messages(pending_job_ids, when))
# Create new messages to create recipe nodes
conditions = []
recipe_jobs = []
subrecipes = []
for node_name, node_def in nodes_to_create.items():
process_input = False
if node_name in nodes_to_process_input:
process_input = True
del nodes_to_process_input[node_name]
if node_def.node_type == ConditionNodeDefinition.NODE_TYPE:
condition = Condition(node_name, process_input)
conditions.append(condition)
elif node_def.node_type == JobNodeDefinition.NODE_TYPE:
job = RecipeJob(node_def.job_type_name,
node_def.job_type_version,
node_def.revision_num, node_name,
process_input)
recipe_jobs.append(job)
elif node_def.node_type == RecipeNodeDefinition.NODE_TYPE:
subrecipe = SubRecipe(node_def.recipe_type_name,
node_def.revision_num, node_name,
process_input)
subrecipes.append(subrecipe)
if len(conditions):
logger.info('Found %d condition(s) to create for this recipe',
len(conditions))
self.new_messages.extend(
create_conditions_messages(recipe_model, conditions))
if len(recipe_jobs):
logger.info('Found %d job(s) to create for this recipe',
len(recipe_jobs))
self.new_messages.extend(
create_jobs_messages_for_recipe(recipe_model, recipe_jobs))
if len(subrecipes):
logger.info('Found %d sub-recipe(s) to create for this recipe',
len(subrecipes))
self.new_messages.extend(
create_subrecipes_messages(recipe_model,
subrecipes,
forced_nodes=self.forced_nodes))
# Create new messages for processing recipe node input
process_condition_ids = []
process_job_ids = []
process_recipe_ids = []
for node_name, node in nodes_to_process_input.items():
if node.node_type == ConditionNodeDefinition.NODE_TYPE:
process_condition_ids.append(node.condition.id)
elif node.node_type == JobNodeDefinition.NODE_TYPE:
process_job_ids.append(node.job.id)
elif node.node_type == RecipeNodeDefinition.NODE_TYPE:
process_recipe_ids.append(node.recipe.id)
if len(process_condition_ids):
logger.info('Found %d condition(s) to process their input',
len(process_condition_ids))
self.new_messages.extend(
create_process_condition_messages(process_condition_ids))
if len(process_job_ids):
logger.info(
'Found %d job(s) to process their input and move to the queue',
len(process_job_ids))
self.new_messages.extend(
create_process_job_input_messages(process_job_ids))
if len(process_recipe_ids):
logger.info(
'Found %d sub-recipe(s) to process their input and begin processing',
len(process_recipe_ids))
self.new_messages.extend(
create_process_recipe_input_messages(process_recipe_ids))
return True
def test_execute(self):
"""Tests calling UpdateRecipe.execute() successfully"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_a = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
recipe_b = recipe_test_utils.create_recipe()
job_c = job_test_utils.create_job(status='PENDING')
job_d = job_test_utils.create_job(status='BLOCKED')
recipe_type_e = recipe_test_utils.create_recipe_type_v6()
job_type_f = job_test_utils.create_seed_job_type()
job_type_k = job_test_utils.create_seed_job_type()
job_g = job_test_utils.create_job(status='FAILED', input=data_dict)
job_h = job_test_utils.create_job(status='PENDING')
condition_i = recipe_test_utils.create_recipe_condition(save=True)
definition = RecipeDefinition(Interface())
definition.add_job_node('node_a', job_a.job_type.name, job_a.job_type.version, job_a.job_type_rev.revision_num)
definition.add_recipe_node('node_b', recipe_b.recipe_type.name, recipe_b.recipe_type.revision_num)
definition.add_job_node('node_c', job_c.job_type.name, job_c.job_type.version, job_c.job_type_rev.revision_num)
definition.add_job_node('node_d', job_d.job_type.name, job_d.job_type.version, job_d.job_type_rev.revision_num)
definition.add_recipe_node('node_e', recipe_type_e.name, recipe_type_e.revision_num)
definition.add_job_node('node_f', job_type_f.name, job_type_f.version, job_type_f.revision_num)
definition.add_job_node('node_g', job_g.job_type.name, job_g.job_type.version, job_g.job_type_rev.revision_num)
definition.add_job_node('node_h', job_h.job_type.name, job_h.job_type.version, job_h.job_type_rev.revision_num)
definition.add_condition_node('node_i', Interface(), DataFilter()) #True
definition.add_condition_node('node_j', Interface(), DataFilter()) #True
definition.add_job_node('node_k', job_type_k.name, job_type_k.version, job_type_k.revision_num)
definition.add_dependency('node_a', 'node_c')
definition.add_dependency('node_a', 'node_e')
definition.add_dependency('node_a', 'node_g')
definition.add_dependency('node_c', 'node_d')
definition.add_dependency('node_e', 'node_f')
definition.add_dependency('node_g', 'node_h')
definition.add_dependency('node_a', 'node_i')
definition.add_dependency('node_a', 'node_j')
definition.add_dependency('node_i', 'node_k')
definition.add_dependency('node_j', 'node_k')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', job=job_a, save=False)
node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_b', sub_recipe=recipe_b,
save=False)
node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_c', job=job_c, save=False)
node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_d', job=job_d, save=False)
node_g = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_g', job=job_g, save=False)
node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_h', job=job_h, save=False)
node_i = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_i', condition=condition_i,
save=False)
RecipeNode.objects.bulk_create([node_a, node_b, node_c, node_d, node_g, node_h, node_i])
forced_nodes = ForcedNodes()
forced_nodes_e = ForcedNodes()
forced_nodes_e.set_all_nodes()
forced_nodes.add_subrecipe('node_e', forced_nodes_e)
# Create and execute message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
result = message.execute()
self.assertTrue(result)
self.assertEqual(len(message.new_messages), 8)
# Check messages
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipe.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Make sure the same messages are returned
self.assertEqual(len(message.new_messages), 8)
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
def test_execute_subrecipes_superseded(self):
"""Tests calling CreateRecipes.execute() successfully when creating sub-recipes that supersede other sub-recipes
"""
# Creates definitions for sub-recipe A and sub-recipe B
batch = batch_test_utils.create_batch()
event = trigger_test_utils.create_trigger_event()
top_recipe_type = recipe_test_utils.create_recipe_type()
job_type_a_1 = job_test_utils.create_seed_job_type()
job_type_a_2 = job_test_utils.create_seed_job_type()
sub_definition_a = RecipeDefinition(Interface())
sub_definition_a.add_job_node('node_1', job_type_a_1.name,
job_type_a_1.version,
job_type_a_1.revision_num)
sub_definition_a.add_job_node('node_2', job_type_a_2.name,
job_type_a_2.version,
job_type_a_2.revision_num)
sub_definition_a.add_dependency('node_1', 'node_2')
sub_definition_a_dict = convert_recipe_definition_to_v6_json(
sub_definition_a).get_dict()
recipe_type_a = recipe_test_utils.create_recipe_type(
definition=sub_definition_a_dict)
job_type_b_x = job_test_utils.create_seed_job_type()
job_type_b_y = job_test_utils.create_seed_job_type()
recipe_type_b_z = recipe_test_utils.create_recipe_type()
sub_definition_b = RecipeDefinition(Interface())
sub_definition_b.add_job_node('node_x', job_type_b_x.name,
job_type_b_x.version,
job_type_b_x.revision_num)
sub_definition_b.add_job_node('node_y', job_type_b_y.name,
job_type_b_y.version,
job_type_b_y.revision_num)
sub_definition_b.add_recipe_node('node_z', recipe_type_b_z.name,
recipe_type_b_z.revision_num)
sub_definition_b.add_dependency('node_x', 'node_z')
sub_definition_b.add_dependency('node_y', 'node_z')
sub_definition_b_dict = convert_recipe_definition_to_v6_json(
sub_definition_b).get_dict()
recipe_type_b = recipe_test_utils.create_recipe_type(
definition=sub_definition_b_dict)
# Create previous recipe containing sub-recipe A and B in order to be superseded
prev_recipe_a = recipe_test_utils.create_recipe(
recipe_type=recipe_type_a, save=False)
prev_job_a_1 = job_test_utils.create_job(job_type=job_type_a_1,
save=False)
prev_job_a_2 = job_test_utils.create_job(job_type=job_type_a_2,
save=False)
prev_recipe_b = recipe_test_utils.create_recipe(
recipe_type=recipe_type_b, save=False)
prev_job_b_x = job_test_utils.create_job(job_type=job_type_b_x,
save=False)
prev_job_b_y = job_test_utils.create_job(job_type=job_type_b_y,
save=False)
prev_recipe_b_z = recipe_test_utils.create_recipe(
recipe_type=recipe_type_b_z, save=False)
prev_top_recipe = recipe_test_utils.create_recipe(
recipe_type=top_recipe_type, save=False)
new_top_recipe = recipe_test_utils.create_recipe(
recipe_type=top_recipe_type, save=False)
Job.objects.bulk_create(
[prev_job_a_1, prev_job_a_2, prev_job_b_x, prev_job_b_y])
Recipe.objects.bulk_create([
prev_recipe_a, prev_recipe_b, prev_recipe_b_z, prev_top_recipe,
new_top_recipe
])
recipe_node_a = recipe_test_utils.create_recipe_node(
recipe=prev_top_recipe,
sub_recipe=prev_recipe_a,
node_name='node_a',
save=False)
recipe_node_a_1 = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_a,
job=prev_job_a_1,
node_name='node_1',
save=False)
recipe_node_a_2 = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_a,
job=prev_job_a_2,
node_name='node_2',
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=prev_top_recipe,
sub_recipe=prev_recipe_b,
node_name='node_b',
save=False)
recipe_node_b_x = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_b,
job=prev_job_b_x,
node_name='node_x',
save=False)
recipe_node_b_y = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_b,
job=prev_job_b_y,
node_name='node_y',
save=False)
recipe_node_b_z = recipe_test_utils.create_recipe_node(
recipe=prev_recipe_b,
sub_recipe=prev_recipe_b_z,
node_name='node_z',
save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_a_1, recipe_node_a_2, recipe_node_b,
recipe_node_b_x, recipe_node_b_y, recipe_node_b_z
])
# Create message to create sub-recipes A and B for new_top_recipe which supersedes prev_top_recipe
sub_recipes = [
SubRecipe(recipe_type_a.name, recipe_type_a.revision_num, 'node_a',
True),
SubRecipe(recipe_type_b.name, recipe_type_b.revision_num, 'node_b',
False)
]
forced_nodes = ForcedNodes()
sub_forced_nodes_b = ForcedNodes()
sub_forced_nodes_y = ForcedNodes()
sub_forced_nodes_b.add_subrecipe('node_y', sub_forced_nodes_y)
forced_nodes.add_subrecipe('node_b', sub_forced_nodes_b)
message = create_subrecipes_messages(
new_top_recipe.id,
new_top_recipe.root_superseded_recipe_id,
sub_recipes,
event.id,
superseded_recipe_id=prev_top_recipe.id,
forced_nodes=forced_nodes,
batch_id=batch.id)[0]
# Execute message
result = message.execute()
self.assertTrue(result)
# Check for new sub-recipes
qry = RecipeNode.objects.select_related('sub_recipe')
recipe_nodes = qry.filter(
recipe_id=new_top_recipe.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_a')
self.assertEqual(recipe_nodes[1].node_name, 'node_b')
sub_recipe_a = recipe_nodes[0].sub_recipe
sub_recipe_b = recipe_nodes[1].sub_recipe
self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id)
self.assertEqual(sub_recipe_a.superseded_recipe_id, prev_recipe_a.id)
self.assertEqual(sub_recipe_a.root_superseded_recipe_id,
prev_recipe_a.id)
self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id)
self.assertEqual(sub_recipe_b.superseded_recipe_id, prev_recipe_b.id)
self.assertEqual(sub_recipe_b.root_superseded_recipe_id,
prev_recipe_b.id)
# Check for sub-recipes to contain correct copied nodes
# Nodes 1 and 2 in sub-recipe A should be copied
recipe_nodes = RecipeNode.objects.select_related('job').filter(
recipe_id=sub_recipe_a.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_1')
self.assertFalse(recipe_nodes[0].is_original)
self.assertEqual(recipe_nodes[0].job_id, prev_job_a_1.id)
self.assertEqual(recipe_nodes[1].node_name, 'node_2')
self.assertFalse(recipe_nodes[1].is_original)
self.assertEqual(recipe_nodes[1].job_id, prev_job_a_2.id)
# Node X in sub-recipe B should be copied
recipe_nodes = RecipeNode.objects.select_related('sub_recipe').filter(
recipe_id=sub_recipe_b.id)
self.assertEqual(len(recipe_nodes), 1)
self.assertEqual(recipe_nodes[0].node_name, 'node_x')
self.assertFalse(recipe_nodes[0].is_original)
# Should be four messages, two for superseding recipe nodes, one for processing recipe input, and one for
# updating metrics for the recipe containing the new sub-recipes
self.assertEqual(len(message.new_messages), 4)
supersede_recipe_a_msg = None
supersede_recipe_b_msg = None
process_recipe_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'supersede_recipe_nodes':
if msg._recipe_ids[0] == prev_recipe_a.id:
supersede_recipe_a_msg = msg
if msg._recipe_ids[0] == prev_recipe_b.id:
supersede_recipe_b_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(supersede_recipe_a_msg)
self.assertIsNotNone(supersede_recipe_b_msg)
self.assertIsNotNone(process_recipe_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message for superseding previous sub-recipe A
self.assertFalse(supersede_recipe_a_msg.supersede_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_jobs, set())
self.assertSetEqual(supersede_recipe_a_msg.supersede_subrecipes, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_a_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_recursive, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_recursive, set())
# Check message for superseding previous sub-recipe B
self.assertFalse(supersede_recipe_b_msg.supersede_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_jobs, {'node_y'})
self.assertSetEqual(supersede_recipe_b_msg.supersede_subrecipes,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_b_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_recursive,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_recursive, set())
# Check message to process recipe input for new sub-recipe A
self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id)
# Check message to update recipe metrics for the recipe containing the new sub-recipes
self.assertListEqual(update_metrics_msg._recipe_ids,
[new_top_recipe.id])
# Test executing message again
message_json_dict = message.to_json()
message = CreateRecipes.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Check sub-recipes to make sure we didn't create them again a second time
qry = RecipeNode.objects.select_related('sub_recipe')
recipe_nodes = qry.filter(
recipe_id=new_top_recipe.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_a')
self.assertEqual(recipe_nodes[1].node_name, 'node_b')
sub_recipe_a = recipe_nodes[0].sub_recipe
sub_recipe_b = recipe_nodes[1].sub_recipe
self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id)
self.assertEqual(sub_recipe_a.superseded_recipe_id, prev_recipe_a.id)
self.assertEqual(sub_recipe_a.root_superseded_recipe_id,
prev_recipe_a.id)
self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id)
self.assertEqual(sub_recipe_b.superseded_recipe_id, prev_recipe_b.id)
self.assertEqual(sub_recipe_b.root_superseded_recipe_id,
prev_recipe_b.id)
# Check for sub-recipes to contain correct copied nodes
# Nodes 1 and 2 in sub-recipe A should be copied
recipe_nodes = RecipeNode.objects.select_related('job').filter(
recipe_id=sub_recipe_a.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_1')
self.assertFalse(recipe_nodes[0].is_original)
self.assertEqual(recipe_nodes[0].job_id, prev_job_a_1.id)
self.assertEqual(recipe_nodes[1].node_name, 'node_2')
self.assertFalse(recipe_nodes[1].is_original)
self.assertEqual(recipe_nodes[1].job_id, prev_job_a_2.id)
# Node X in sub-recipe B should be copied
recipe_nodes = RecipeNode.objects.select_related('sub_recipe').filter(
recipe_id=sub_recipe_b.id)
self.assertEqual(len(recipe_nodes), 1)
self.assertEqual(recipe_nodes[0].node_name, 'node_x')
self.assertFalse(recipe_nodes[0].is_original)
# Check messages again
# Should be four messages, two for superseding recipe nodes, one for processing recipe input, and one for
# updating metrics for the recipe containing the new sub-recipes
self.assertEqual(len(message.new_messages), 4)
supersede_recipe_a_msg = None
supersede_recipe_b_msg = None
process_recipe_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'supersede_recipe_nodes':
if msg._recipe_ids[0] == prev_recipe_a.id:
supersede_recipe_a_msg = msg
if msg._recipe_ids[0] == prev_recipe_b.id:
supersede_recipe_b_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(supersede_recipe_a_msg)
self.assertIsNotNone(supersede_recipe_b_msg)
self.assertIsNotNone(process_recipe_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message for superseding previous sub-recipe A
self.assertFalse(supersede_recipe_a_msg.supersede_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_jobs, set())
self.assertSetEqual(supersede_recipe_a_msg.supersede_subrecipes, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_a_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.supersede_recursive, set())
self.assertFalse(supersede_recipe_a_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_a_msg.unpublish_recursive, set())
# Check message for superseding previous sub-recipe B
self.assertFalse(supersede_recipe_b_msg.supersede_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_jobs, {'node_y'})
self.assertSetEqual(supersede_recipe_b_msg.supersede_subrecipes,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_jobs, set())
self.assertFalse(supersede_recipe_b_msg.supersede_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.supersede_recursive,
{'node_z'})
self.assertFalse(supersede_recipe_b_msg.unpublish_recursive_all)
self.assertSetEqual(supersede_recipe_b_msg.unpublish_recursive, set())
# Check message to process recipe input for new sub-recipe A
self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id)
# Check message to update recipe metrics for the recipe containing the new sub-recipes
self.assertListEqual(update_metrics_msg._recipe_ids,
[new_top_recipe.id])
def test_execute_subrecipes(self):
"""Tests calling CreateRecipes.execute() successfully when creating sub-recipes"""
# Creates definitions for sub-recipe A and sub-recipe B
event = trigger_test_utils.create_trigger_event()
top_recipe_type = recipe_test_utils.create_recipe_type()
job_type_a_1 = job_test_utils.create_seed_job_type()
job_type_a_2 = job_test_utils.create_seed_job_type()
sub_definition_a = RecipeDefinition(Interface())
sub_definition_a.add_job_node('node_1', job_type_a_1.name,
job_type_a_1.version,
job_type_a_1.revision_num)
sub_definition_a.add_job_node('node_2', job_type_a_2.name,
job_type_a_2.version,
job_type_a_2.revision_num)
sub_definition_a.add_dependency('node_1', 'node_2')
sub_definition_a_dict = convert_recipe_definition_to_v6_json(
sub_definition_a).get_dict()
recipe_type_a = recipe_test_utils.create_recipe_type(
definition=sub_definition_a_dict)
job_type_b_x = job_test_utils.create_seed_job_type()
job_type_b_y = job_test_utils.create_seed_job_type()
recipe_type_b_z = recipe_test_utils.create_recipe_type()
sub_definition_b = RecipeDefinition(Interface())
sub_definition_b.add_job_node('node_x', job_type_b_x.name,
job_type_b_x.version,
job_type_b_x.revision_num)
sub_definition_b.add_job_node('node_y', job_type_b_y.name,
job_type_b_y.version,
job_type_b_y.revision_num)
sub_definition_b.add_recipe_node('node_z', recipe_type_b_z.name,
recipe_type_b_z.revision_num)
sub_definition_b.add_dependency('node_x', 'node_z')
sub_definition_b.add_dependency('node_y', 'node_z')
sub_definition_b_dict = convert_recipe_definition_to_v6_json(
sub_definition_b).get_dict()
recipe_type_b = recipe_test_utils.create_recipe_type(
definition=sub_definition_b_dict)
top_recipe = recipe_test_utils.create_recipe(
recipe_type=top_recipe_type, save=True)
# Create message to create sub-recipes A and B for top_recipe which supersedes prev_top_recipe
sub_recipes = [
SubRecipe(recipe_type_a.name, recipe_type_a.revision_num, 'node_a',
True),
SubRecipe(recipe_type_b.name, recipe_type_b.revision_num, 'node_b',
False)
]
message = create_subrecipes_messages(
top_recipe.id, top_recipe.root_superseded_recipe_id, sub_recipes,
event.id)[0]
# Execute message
result = message.execute()
self.assertTrue(result)
# Check for new sub-recipes
qry = RecipeNode.objects.select_related('sub_recipe')
recipe_nodes = qry.filter(
recipe_id=top_recipe.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_a')
self.assertEqual(recipe_nodes[1].node_name, 'node_b')
sub_recipe_a = recipe_nodes[0].sub_recipe
sub_recipe_b = recipe_nodes[1].sub_recipe
self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id)
self.assertIsNone(sub_recipe_a.superseded_recipe_id)
self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id)
self.assertIsNone(sub_recipe_b.superseded_recipe_id)
# Should be two messages, one for processing recipe input, and one for updating metrics for the recipe
# containing the new sub-recipes
self.assertEqual(len(message.new_messages), 2)
process_recipe_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(process_recipe_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message to process recipe input for new sub-recipe A
self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id)
# Check message to update recipe metrics for the recipe containing the new sub-recipes
self.assertListEqual(update_metrics_msg._recipe_ids, [top_recipe.id])
# Test executing message again
message_json_dict = message.to_json()
message = CreateRecipes.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Check sub-recipes to make sure we didn't create them again a second time
qry = RecipeNode.objects.select_related('sub_recipe')
recipe_nodes = qry.filter(
recipe_id=top_recipe.id).order_by('node_name')
self.assertEqual(len(recipe_nodes), 2)
self.assertEqual(recipe_nodes[0].node_name, 'node_a')
self.assertEqual(recipe_nodes[1].node_name, 'node_b')
sub_recipe_a = recipe_nodes[0].sub_recipe
sub_recipe_b = recipe_nodes[1].sub_recipe
self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id)
self.assertIsNone(sub_recipe_a.superseded_recipe_id)
self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id)
self.assertIsNone(sub_recipe_b.superseded_recipe_id)
# Check messages again
# Should be two messages, one for processing recipe input, and one for updating metrics for the recipe
# containing the new sub-recipes
self.assertEqual(len(message.new_messages), 2)
process_recipe_input_msg = None
update_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
elif msg.type == 'update_recipe_metrics':
update_metrics_msg = msg
self.assertIsNotNone(process_recipe_input_msg)
self.assertIsNotNone(update_metrics_msg)
# Check message to process recipe input for new sub-recipe A
self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id)
# Check message to update recipe metrics for the recipe containing the new sub-recipes
self.assertListEqual(update_metrics_msg._recipe_ids, [top_recipe.id])