def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
recipe = Recipe.objects.get_recipe_with_interfaces(self.recipe_id)
if not recipe.has_input():
if not recipe.recipe:
logger.error('Recipe %d has no input and is not in a recipe. Message will not re-run.', self.recipe_id)
return True
try:
self._generate_input_data_from_recipe(recipe)
except InvalidData:
msg = 'Recipe created invalid input data for sub-recipe %d. Message will not re-run.'
logger.exception(msg, self.recipe_id)
return True
# Lock recipe model and process recipe's input data
with transaction.atomic():
recipe = Recipe.objects.get_locked_recipe(self.recipe_id)
root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id
Recipe.objects.process_recipe_input(recipe)
# Create message to update the recipe
from recipe.messages.update_recipe import create_update_recipe_message
logger.info('Processed input for recipe %d, sending message to update recipe', self.recipe_id)
self.new_messages.append(create_update_recipe_message(root_recipe_id, forced_nodes=self.forced_nodes))
return True
def test_json(self):
"""Tests converting an UpdateRecipe message to and from JSON"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_failed = job_test_utils.create_job(status='FAILED', input=data_dict)
job_pending = job_test_utils.create_job(status='PENDING')
definition = RecipeDefinition(Interface())
definition.add_job_node('job_failed', job_failed.job_type.name, job_failed.job_type.version,
job_failed.job_type_rev.revision_num)
definition.add_job_node('job_pending', job_pending.job_type.name, job_pending.job_type.version,
job_pending.job_type_rev.revision_num)
definition.add_dependency('job_failed', 'job_pending')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_failed', job=job_failed)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_pending', job=job_pending)
# Create message
message = create_update_recipe_message(recipe.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to set job_pending to BLOCKED
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'blocked_jobs')
self.assertListEqual(msg._blocked_job_ids, [job_pending.id])
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
condition = RecipeCondition.objects.get_condition_with_interfaces(
self.condition_id)
if not condition.is_processed:
definition = condition.recipe.recipe_type_rev.get_definition()
# Get condition data from dependencies in the recipe
recipe_input_data = condition.recipe.get_input_data()
node_outputs = RecipeNode.objects.get_recipe_node_outputs(
condition.recipe_id)
for node_output in node_outputs.values():
if node_output.node_type == 'condition' and node_output.id == condition.id:
node_name = node_output.node_name
break
# Set data on the condition model
try:
data = definition.generate_node_input_data(
node_name, recipe_input_data, node_outputs)
RecipeCondition.objects.set_condition_data_v6(
condition, data, node_name)
except InvalidData:
logger.exception(
'Recipe %d created invalid input data for condition %d. Message will not re-run.',
condition.recipe_id, self.condition_id)
return True
# Process filter and set whether condition was accepted
data_filter = definition.graph[node_name].data_filter
is_accepted = data_filter.is_data_accepted(data)
RecipeCondition.objects.set_processed(condition.id, is_accepted)
# Log results
filter_str = json.dumps(data_filter.filter_list,
sort_keys=True,
indent=4,
separators=(',', ': '))
data_str = json.dumps(convert_data_to_v6_json(data).get_dict(),
sort_keys=True,
indent=4,
separators=(',', ': '))
logger.info(
'Condition %d (recipe %d at %s) evaluated to %s:\nCondition: %s\nInput Data: %s',
condition.id, condition.recipe_id, node_name, is_accepted,
filter_str, data_str)
# Create message to update the condition's recipe
from recipe.messages.update_recipe import create_update_recipe_message
root_recipe_id = condition.recipe.root_recipe_id if condition.recipe.root_recipe_id else condition.recipe_id
logger.info(
'Processed data for condition %d, sending message to update recipe %d',
self.condition_id, root_recipe_id)
self.new_messages.append(create_update_recipe_message(root_recipe_id))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
for recipe in Recipe.objects.filter(id__in=self._recipe_ids):
root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id
self.new_messages.append(create_update_recipe_message(root_recipe_id))
logger.info('Found %d message(s) to update recipes', len(self.new_messages))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
condition = RecipeCondition.objects.get_condition_with_interfaces(
self.condition_id)
if not condition.is_processed:
definition = condition.recipe.recipe_type_rev.get_definition()
# Get condition data from dependencies in the recipe
recipe_input_data = condition.recipe.get_input_data()
node_outputs = RecipeNode.objects.get_recipe_node_outputs(
condition.recipe_id)
for node_output in node_outputs.values():
if node_output.node_type == 'condition' and node_output.id == condition.id:
node_name = node_output.node_name
break
# Set data on the condition model
try:
data = definition.generate_node_input_data(
node_name, recipe_input_data, node_outputs)
RecipeCondition.objects.set_condition_data_v6(
condition, data, node_name)
except InvalidData:
logger.exception(
'Recipe created invalid input data for condition %d. Message will not re-run.',
self.condition_id)
return True
# Process filter and set whether condition was accepted
data_filter = definition.graph[node_name].data_filter
is_accepted = data_filter.is_data_accepted(data)
RecipeCondition.objects.set_processed(condition.id, is_accepted)
# Create message to update the condition's recipe
from recipe.messages.update_recipe import create_update_recipe_message
root_recipe_id = condition.recipe.root_recipe_id if condition.recipe.root_recipe_id else condition.recipe_id
logger.info(
'Processed data for condition %d, sending message to update recipe',
self.condition_id)
self.new_messages.append(create_update_recipe_message(root_recipe_id))
return True
def test_json_forced_nodes(self):
"""Tests converting an UpdateRecipe message to and from JSON when forced nodes are provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_completed = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('job_completed', job_completed.job_type.name, job_completed.job_type.version,
job_completed.job_type_rev.revision_num)
definition.add_recipe_node('the_sub_recipe', sub_recipe_type.name, sub_recipe_type.revision_num)
definition.add_dependency('job_completed', 'the_sub_recipe')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_completed', job=job_completed)
forced_nodes = ForcedNodes()
sub_forced_nodes = ForcedNodes()
sub_forced_nodes.set_all_nodes()
forced_nodes.add_subrecipe('the_sub_recipe', sub_forced_nodes)
# Create message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to create sub-recipe
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'create_recipes')
self.assertEqual(msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(msg.recipe_id, recipe.id)
self.assertEqual(msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(msg.superseded_recipe_id)
sub = SubRecipe(sub_recipe_type.name, sub_recipe_type.revision_num, 'the_sub_recipe', True)
self.assertListEqual(msg.sub_recipes, [sub])
def _create_messages(self, new_recipes):
"""Creates any messages resulting from the new recipes
:param new_recipes: The list of new recipe models
:type new_recipes: list
"""
forced_nodes_by_id = {} # {Recipe ID: Forced nodes}
# Send supersede_recipe_nodes messages if new recipes are superseding old ones
for recipe_diff in self._recipe_diffs:
recipe_ids = []
supersede_jobs = set()
supersede_subrecipes = set()
unpublish_jobs = set()
supersede_recursive = set()
unpublish_recursive = set()
# Gather up superseded recipe IDs for this diff
for recipe_pair in recipe_diff.recipe_pairs:
recipe_ids.append(recipe_pair.superseded_recipe.id)
# Supersede applicable jobs and sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_supersede().values(
):
if node_diff.node_type == JobNodeDefinition.NODE_TYPE:
supersede_jobs.add(node_diff.name)
elif node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
supersede_subrecipes.add(node_diff.name)
# Recursively supersede applicable sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_recursively_supersede(
).values():
if node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
supersede_recursive.add(node_diff.name)
# Update forced nodes so that the new sub-recipes know to force reprocess all nodes
if not recipe_diff.diff.forced_nodes:
recipe_diff.diff.forced_nodes = ForcedNodes()
force_all = ForcedNodes()
force_all.set_all_nodes()
recipe_diff.diff.forced_nodes.add_subrecipe(
node_diff.name, force_all)
# Unpublish applicable jobs and recursively unpublish applicable sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_unpublish().values(
):
if node_diff.node_type == JobNodeDefinition.NODE_TYPE:
unpublish_jobs.add(node_diff.name)
elif node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
unpublish_recursive.add(node_diff.name)
if supersede_jobs or supersede_subrecipes or unpublish_jobs or supersede_recursive or unpublish_recursive:
msgs = create_supersede_recipe_nodes_messages(
recipe_ids, self._when, supersede_jobs,
supersede_subrecipes, unpublish_jobs, supersede_recursive,
unpublish_recursive)
self.new_messages.extend(msgs)
if recipe_diff.diff.forced_nodes:
# Store the forced nodes for this diff by every new recipe ID in the diff
ids = [pair.new_recipe.id for pair in recipe_diff.recipe_pairs]
forced_nodes_by_id.update({
recipe_id: recipe_diff.diff.forced_nodes
for recipe_id in ids
})
# Send messages to further process/update the new recipes
if self.create_recipes_type == NEW_RECIPE_TYPE or self.create_recipes_type == REPROCESS_TYPE:
msgs = create_process_recipe_input_messages(
[r.id for r in new_recipes], forced_nodes=self.forced_nodes)
self.new_messages.extend(msgs)
elif self.create_recipes_type == SUB_RECIPE_TYPE:
from recipe.messages.update_recipe import create_update_recipe_message
for new_recipe in new_recipes:
process_input = self._process_input.get(new_recipe.id, False)
forced_nodes = forced_nodes_by_id.get(new_recipe.id, None)
if new_recipe.has_input() or process_input:
# This new recipe is all ready to have its input processed
msg = create_process_recipe_input_messages(
[new_recipe.id], forced_nodes=forced_nodes)[0]
else:
# Recipe not ready for its input yet, but send update_recipe for it to create its nodes awhile
root_id = new_recipe.id
if new_recipe.root_superseded_recipe_id:
root_id = new_recipe.root_superseded_recipe_id
msg = create_update_recipe_message(
root_id, forced_nodes=forced_nodes)
self.new_messages.append(msg)
if self.recipe_id:
# Update the metrics for the recipe containing the new sub-recipes we just created
self.new_messages.extend(
create_update_recipe_metrics_messages([self.recipe_id]))
def test_execute(self):
"""Tests calling UpdateRecipe.execute() successfully"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_a = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
recipe_b = recipe_test_utils.create_recipe()
job_c = job_test_utils.create_job(status='PENDING')
job_d = job_test_utils.create_job(status='BLOCKED')
recipe_type_e = recipe_test_utils.create_recipe_type_v6()
job_type_f = job_test_utils.create_seed_job_type()
job_type_k = job_test_utils.create_seed_job_type()
job_g = job_test_utils.create_job(status='FAILED', input=data_dict)
job_h = job_test_utils.create_job(status='PENDING')
condition_i = recipe_test_utils.create_recipe_condition(save=True)
definition = RecipeDefinition(Interface())
definition.add_job_node('node_a', job_a.job_type.name, job_a.job_type.version, job_a.job_type_rev.revision_num)
definition.add_recipe_node('node_b', recipe_b.recipe_type.name, recipe_b.recipe_type.revision_num)
definition.add_job_node('node_c', job_c.job_type.name, job_c.job_type.version, job_c.job_type_rev.revision_num)
definition.add_job_node('node_d', job_d.job_type.name, job_d.job_type.version, job_d.job_type_rev.revision_num)
definition.add_recipe_node('node_e', recipe_type_e.name, recipe_type_e.revision_num)
definition.add_job_node('node_f', job_type_f.name, job_type_f.version, job_type_f.revision_num)
definition.add_job_node('node_g', job_g.job_type.name, job_g.job_type.version, job_g.job_type_rev.revision_num)
definition.add_job_node('node_h', job_h.job_type.name, job_h.job_type.version, job_h.job_type_rev.revision_num)
definition.add_condition_node('node_i', Interface(), DataFilter()) #True
definition.add_condition_node('node_j', Interface(), DataFilter()) #True
definition.add_job_node('node_k', job_type_k.name, job_type_k.version, job_type_k.revision_num)
definition.add_dependency('node_a', 'node_c')
definition.add_dependency('node_a', 'node_e')
definition.add_dependency('node_a', 'node_g')
definition.add_dependency('node_c', 'node_d')
definition.add_dependency('node_e', 'node_f')
definition.add_dependency('node_g', 'node_h')
definition.add_dependency('node_a', 'node_i')
definition.add_dependency('node_a', 'node_j')
definition.add_dependency('node_i', 'node_k')
definition.add_dependency('node_j', 'node_k')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', job=job_a, save=False)
node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_b', sub_recipe=recipe_b,
save=False)
node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_c', job=job_c, save=False)
node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_d', job=job_d, save=False)
node_g = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_g', job=job_g, save=False)
node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_h', job=job_h, save=False)
node_i = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_i', condition=condition_i,
save=False)
RecipeNode.objects.bulk_create([node_a, node_b, node_c, node_d, node_g, node_h, node_i])
forced_nodes = ForcedNodes()
forced_nodes_e = ForcedNodes()
forced_nodes_e.set_all_nodes()
forced_nodes.add_subrecipe('node_e', forced_nodes_e)
# Create and execute message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
result = message.execute()
self.assertTrue(result)
self.assertEqual(len(message.new_messages), 8)
# Check messages
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipe.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Make sure the same messages are returned
self.assertEqual(len(message.new_messages), 8)
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
