def test_json_forced_nodes(self):
"""Tests coverting a ProcessRecipeInput message to and from JSON with forced nodes provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
recipe = recipe_test_utils.create_recipe(input=data_dict)
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
# Create message
message = create_process_recipe_input_messages(
[recipe.id], forced_nodes=forced_nodes)[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessRecipeInput.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'update_recipe')
self.assertEqual(msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(msg.forced_nodes).get_dict(),
forced_nodes_dict)
# Recipe should have input_file_size set to 0 (no input files)
self.assertEqual(recipe.input_file_size, 0.0)
def queue_new_recipe_v6(self,
recipe_type,
recipe_input,
event,
ingest_event=None,
recipe_config=None,
batch_id=None,
superseded_recipe=None):
"""Creates a new recipe for the given type and data. and queues any of its jobs that are ready to run. If the
new recipe is superseding an old recipe, superseded_recipe, delta, and superseded_jobs must be provided and the
caller must have obtained a model lock on all job models in superseded_jobs and on the superseded_recipe model.
All database changes occur in an atomic transaction.
:param recipe_type: The type of the new recipe to create
:type recipe_type: :class:`recipe.models.RecipeType`
:param recipe_input: The recipe data to run on, should be None if superseded_recipe is provided
:type recipe_input: :class:`data.data.data.data`
:param event: The event that triggered the creation of this recipe
:type event: :class:`trigger.models.TriggerEvent`
:param recipe_config: config of the recipe, possibly None
:type recipe_config: :class:`recipe.configuration.configuration.RecipeConfiguration`
:param batch_id: The ID of the batch that contains this recipe
:type batch_id: int
:param superseded_recipe: The recipe that the created recipe is superseding, possibly None
:type superseded_recipe: :class:`recipe.models.Recipe`
:returns: New recipe type
:rtype: :class:`recipe.models.Recipe`
:raises :class:`recipe.configuration.data.exceptions.InvalidRecipeData`: If the recipe data is invalid
:raises :class:`recipe.exceptions.InactiveRecipeType`: If the recipe type is inactive
"""
recipe_type_rev = RecipeTypeRevision.objects.get_revision(
recipe_type.name, recipe_type.revision_num)
ingest_event_pk = None
event_pk = None
if ingest_event:
ingest_event_pk = ingest_event.pk
if event:
event_pk = event.pk
recipe = None
with transaction.atomic():
recipe = Recipe.objects.create_recipe_v6(
recipe_type_rev=recipe_type_rev,
event_id=event_pk,
ingest_id=ingest_event_pk,
input_data=recipe_input,
recipe_config=recipe_config,
batch_id=batch_id,
superseded_recipe=superseded_recipe)
recipe.save()
# If root_recipe_id is allowed to be None moving forword then a recipe will never complete.
# The recipe pk is only accessible once the recipe is saved (L#442).
if not recipe.recipe_id and not recipe.root_recipe_id:
recipe.root_recipe_id = recipe.pk
recipe.save()
# This can cause a race condition with a slow DB.
if recipe:
CommandMessageManager().send_messages(
create_process_recipe_input_messages([recipe.id]))
return recipe
def _create_messages(self, new_recipes):
"""Creates any messages resulting from the new recipes
:param new_recipes: The list of new recipe models
:type new_recipes: list
"""
forced_nodes_by_id = {} # {Recipe ID: Forced nodes}
# Send supersede_recipe_nodes messages if new recipes are superseding old ones
for recipe_diff in self._recipe_diffs:
recipe_ids = []
supersede_jobs = set()
supersede_subrecipes = set()
unpublish_jobs = set()
supersede_recursive = set()
unpublish_recursive = set()
# Gather up superseded recipe IDs for this diff
for recipe_pair in recipe_diff.recipe_pairs:
recipe_ids.append(recipe_pair.superseded_recipe.id)
# Supersede applicable jobs and sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_supersede().values(
):
if node_diff.node_type == JobNodeDefinition.NODE_TYPE:
supersede_jobs.add(node_diff.name)
elif node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
supersede_subrecipes.add(node_diff.name)
# Recursively supersede applicable sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_recursively_supersede(
).values():
if node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
supersede_recursive.add(node_diff.name)
# Update forced nodes so that the new sub-recipes know to force reprocess all nodes
if not recipe_diff.diff.forced_nodes:
recipe_diff.diff.forced_nodes = ForcedNodes()
force_all = ForcedNodes()
force_all.set_all_nodes()
recipe_diff.diff.forced_nodes.add_subrecipe(
node_diff.name, force_all)
# Unpublish applicable jobs and recursively unpublish applicable sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_unpublish().values(
):
if node_diff.node_type == JobNodeDefinition.NODE_TYPE:
unpublish_jobs.add(node_diff.name)
elif node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
unpublish_recursive.add(node_diff.name)
if supersede_jobs or supersede_subrecipes or unpublish_jobs or supersede_recursive or unpublish_recursive:
msgs = create_supersede_recipe_nodes_messages(
recipe_ids, self._when, supersede_jobs,
supersede_subrecipes, unpublish_jobs, supersede_recursive,
unpublish_recursive)
self.new_messages.extend(msgs)
if recipe_diff.diff.forced_nodes:
# Store the forced nodes for this diff by every new recipe ID in the diff
ids = [pair.new_recipe.id for pair in recipe_diff.recipe_pairs]
forced_nodes_by_id.update({
recipe_id: recipe_diff.diff.forced_nodes
for recipe_id in ids
})
# Send messages to further process/update the new recipes
if self.create_recipes_type == NEW_RECIPE_TYPE or self.create_recipes_type == REPROCESS_TYPE:
msgs = create_process_recipe_input_messages(
[r.id for r in new_recipes], forced_nodes=self.forced_nodes)
self.new_messages.extend(msgs)
elif self.create_recipes_type == SUB_RECIPE_TYPE:
from recipe.messages.update_recipe import create_update_recipe_message
for new_recipe in new_recipes:
process_input = self._process_input.get(new_recipe.id, False)
forced_nodes = forced_nodes_by_id.get(new_recipe.id, None)
if new_recipe.has_input() or process_input:
# This new recipe is all ready to have its input processed
msg = create_process_recipe_input_messages(
[new_recipe.id], forced_nodes=forced_nodes)[0]
else:
# Recipe not ready for its input yet, but send update_recipe for it to create its nodes awhile
root_id = new_recipe.id
if new_recipe.root_superseded_recipe_id:
root_id = new_recipe.root_superseded_recipe_id
msg = create_update_recipe_message(
root_id, forced_nodes=forced_nodes)
self.new_messages.append(msg)
if self.recipe_id:
# Update the metrics for the recipe containing the new sub-recipes we just created
self.new_messages.extend(
create_update_recipe_metrics_messages([self.recipe_id]))
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
recipe = Recipe.objects.get_recipe_instance_from_root(
self.root_recipe_id)
recipe_model = recipe.recipe_model
when = now()
jobs_to_update = recipe.get_jobs_to_update()
blocked_job_ids = jobs_to_update['BLOCKED']
pending_job_ids = jobs_to_update['PENDING']
nodes_to_create = recipe.get_nodes_to_create()
nodes_to_process_input = recipe.get_nodes_to_process_input()
if not recipe_model.is_completed and recipe.has_completed():
Recipe.objects.complete_recipes([recipe_model.id], when)
# Create new messages for changing job statuses
if len(blocked_job_ids):
logger.info('Found %d job(s) that should transition to BLOCKED',
len(blocked_job_ids))
self.new_messages.extend(
create_blocked_jobs_messages(blocked_job_ids, when))
if len(pending_job_ids):
logger.info('Found %d job(s) that should transition to PENDING',
len(pending_job_ids))
self.new_messages.extend(
create_pending_jobs_messages(pending_job_ids, when))
# Create new messages to create recipe nodes
conditions = []
recipe_jobs = []
subrecipes = []
for node_name, node_def in nodes_to_create.items():
process_input = False
if node_name in nodes_to_process_input:
process_input = True
del nodes_to_process_input[node_name]
if node_def.node_type == ConditionNodeDefinition.NODE_TYPE:
condition = Condition(node_name, process_input)
conditions.append(condition)
elif node_def.node_type == JobNodeDefinition.NODE_TYPE:
job = RecipeJob(node_def.job_type_name,
node_def.job_type_version,
node_def.revision_num, node_name,
process_input)
recipe_jobs.append(job)
elif node_def.node_type == RecipeNodeDefinition.NODE_TYPE:
subrecipe = SubRecipe(node_def.recipe_type_name,
node_def.revision_num, node_name,
process_input)
subrecipes.append(subrecipe)
if len(conditions):
logger.info('Found %d condition(s) to create for this recipe',
len(conditions))
self.new_messages.extend(
create_conditions_messages(recipe_model, conditions))
if len(recipe_jobs):
logger.info('Found %d job(s) to create for this recipe',
len(recipe_jobs))
self.new_messages.extend(
create_jobs_messages_for_recipe(recipe_model, recipe_jobs))
if len(subrecipes):
logger.info('Found %d sub-recipe(s) to create for this recipe',
len(subrecipes))
self.new_messages.extend(
create_subrecipes_messages(recipe_model,
subrecipes,
forced_nodes=self.forced_nodes))
# Create new messages for processing recipe node input
process_condition_ids = []
process_job_ids = []
process_recipe_ids = []
for node_name, node in nodes_to_process_input.items():
if node.node_type == ConditionNodeDefinition.NODE_TYPE:
process_condition_ids.append(node.condition.id)
elif node.node_type == JobNodeDefinition.NODE_TYPE:
process_job_ids.append(node.job.id)
elif node.node_type == RecipeNodeDefinition.NODE_TYPE:
process_recipe_ids.append(node.recipe.id)
if len(process_condition_ids):
logger.info('Found %d condition(s) to process their input',
len(process_condition_ids))
self.new_messages.extend(
create_process_condition_messages(process_condition_ids))
if len(process_job_ids):
logger.info(
'Found %d job(s) to process their input and move to the queue',
len(process_job_ids))
self.new_messages.extend(
create_process_job_input_messages(process_job_ids))
if len(process_recipe_ids):
logger.info(
'Found %d sub-recipe(s) to process their input and begin processing',
len(process_recipe_ids))
self.new_messages.extend(
create_process_recipe_input_messages(process_recipe_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
when = now()
msg_already_run = False
with transaction.atomic():
# Lock recipes
superseded_recipes = Recipe.objects.get_locked_recipes_from_root_old(
self._root_recipe_ids, event_id=self.event_id)
if not superseded_recipes:
# The database transaction has already been completed, just need to resend messages
logger.warning(
'This database transaction appears to have already been executed, will resend messages'
)
msg_already_run = True
recipes = Recipe.objects.filter(
root_superseded_recipe_id__in=self._root_recipe_ids,
event_id=self.event_id)
superseded_recipes = Recipe.objects.get_locked_recipes(
[r.superseded_recipe_id for r in recipes])
# Get revisions and filter out invalid recipes (wrong recipe type)
revisions = RecipeTypeRevision.objects.get_revisions_for_reprocess_old(
superseded_recipes, self.revision_id)
recipe_type = revisions.values()[0].recipe_type
superseded_recipes = [
recipe for recipe in superseded_recipes
if recipe.recipe_type_id == recipe_type.id
]
if len(superseded_recipes) < len(self._root_recipe_ids):
diff = len(self._root_recipe_ids) - len(superseded_recipes)
logger.warning('%d invalid recipes have been filtered out',
diff)
# Create new recipes and supersede old recipes
if not msg_already_run:
recipes = Recipe.objects.create_recipes_for_reprocess(
recipe_type,
revisions,
superseded_recipes,
self.event_id,
batch_id=self.batch_id)
Recipe.objects.bulk_create(recipes)
logger.info('Created %d new recipe(s)', len(recipes))
Recipe.objects.supersede_recipes(
[recipe.id for recipe in superseded_recipes], when)
logger.info('Superseded %d old recipe(s)', len(recipes))
# Handle superseded recipe jobs
recipe_job_ids = RecipeNode.objects.get_recipe_job_ids(
[recipe.id for recipe in superseded_recipes])
messages = self._handle_recipe_jobs(msg_already_run, recipes,
self.revision_id, revisions,
recipe_job_ids, self.job_names,
self.all_jobs, when)
self.new_messages.extend(messages)
# Create messages to handle new recipes
self.new_messages.extend(
create_process_recipe_input_messages(
[recipe.id for recipe in recipes]))
return True
def _create_messages(self, new_recipes):
"""Creates any messages resulting from the new recipes
:param new_recipes: The list of new recipe models
:type new_recipes: list
"""
# Send supersede_recipe_nodes messages if new recipes are superseding old ones
for recipe_diff in self._recipe_diffs:
recipe_ids = []
supersede_jobs = set()
supersede_subrecipes = set()
unpublish_jobs = set()
supersede_recursive = set()
unpublish_recursive = set()
# Gather up superseded recipe IDs for this diff
for recipe_pair in recipe_diff.recipe_pairs:
recipe_ids.append(recipe_pair.superseded_recipe.id)
# Supersede applicable jobs and sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_supersede().values(
):
if node_diff.node_type == JobNodeDefinition.NODE_TYPE:
supersede_jobs.add(node_diff.name)
elif node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
supersede_subrecipes.add(node_diff.name)
# Recursively supersede applicable sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_recursively_supersede(
).values():
if node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
supersede_recursive.add(node_diff.name)
# Unpublish applicable jobs and recursively unpublish applicable sub-recipes
for node_diff in recipe_diff.diff.get_nodes_to_unpublish().values(
):
if node_diff.node_type == JobNodeDefinition.NODE_TYPE:
unpublish_jobs.add(node_diff.name)
elif node_diff.node_type == RecipeNodeDefinition.NODE_TYPE:
unpublish_recursive.add(node_diff.name)
msgs = create_supersede_recipe_nodes_messages(
recipe_ids, self._when, supersede_jobs, supersede_subrecipes,
unpublish_jobs, supersede_recursive, unpublish_recursive)
self.new_messages.extend(msgs)
process_input_recipe_ids = []
update_recipe_ids = []
for new_recipe in new_recipes:
# process_input indicates if new_recipe is a sub-recipe and ready to get its input from its dependencies
process_input = self.recipe_id and self._process_input.get(
new_recipe.id, False)
if new_recipe.has_input() or process_input:
# This new recipe is all ready to have its input processed
process_input_recipe_ids.append(new_recipe.id)
else:
# Recipe not ready for its input yet, but send update_recipe for it to create its nodes awhile
update_recipe_ids.append(new_recipe.id)
self.new_messages.extend(
create_process_recipe_input_messages(process_input_recipe_ids))
# TODO: create messages to update recipes after new update_recipe message is created
# TODO: use get_nodes_to_recursively_supersede() to affect forced nodes passed to process_input_recipe and
# update_recipe messages
if self.recipe_id:
# Update the metrics for the recipe containing the new sub-recipes we just created
self.new_messages.extend(
create_update_recipe_metrics_messages([self.recipe_id]))