def test_execute_with_parent_recipe(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create recipes
recipe = recipe_test_utils.create_recipe()
parent_recipe = recipe_test_utils.create_recipe()
recipe_test_utils.create_recipe_node(recipe=parent_recipe,
node_name='A',
sub_recipe=recipe,
save=True)
# Create message
message = create_purge_recipe_message(recipe_id=recipe.id,
trigger_id=self.trigger.id,
source_file_id=self.file_1.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the parent recipe was sent
msgs = [
msg for msg in message.new_messages if msg.type == 'purge_recipe'
]
self.assertEqual(len(msgs), 1)
for msg in msgs:
self.assertEqual(msg.recipe_id, parent_recipe.id)
# Assert models were deleted
self.assertEqual(Recipe.objects.filter(id=recipe.id).count(), 0)
self.assertEqual(RecipeNode.objects.filter(recipe=recipe).count(), 0)
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
# Kick off purge_source_file for the source file input
self.new_messages.append(create_purge_source_file_message(source_file_id=self.source_file_id,
trigger_id=self.trigger_id))
# Kick off purge_recipe for recipe with node job
parent_recipes = RecipeNode.objects.filter(job__in=self._purge_job_ids, is_original=True)
for recipe_node in parent_recipes:
self.new_messages.append(create_purge_recipe_message(recipe_id=recipe_node.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
with transaction.atomic():
job_exe_queryset = JobExecution.objects.filter(job__in=self._purge_job_ids)
TaskUpdate.objects.filter(job_exe__in=job_exe_queryset).delete()
JobExecutionOutput.objects.filter(job_exe__in=job_exe_queryset).delete()
JobExecutionEnd.objects.filter(job_exe__in=job_exe_queryset).delete()
job_exe_queryset.delete()
FileAncestryLink.objects.filter(job__in=self._purge_job_ids).delete()
BatchJob.objects.filter(job__in=self._purge_job_ids).delete()
RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
Queue.objects.filter(job__in=self._purge_job_ids).delete()
Job.objects.filter(id__in=self._purge_job_ids).delete()
return True
def test_execute_force_stop(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create PurgeResults entry
file_2 = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=file_2.id,
trigger_event=trigger,
force_stop_purge=True)
self.assertEqual(
PurgeResults.objects.values_list(
'num_recipes_deleted',
flat=True).get(trigger_event=trigger.id), 0)
# Create recipes
recipe_type = recipe_test_utils.create_recipe_type()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
# Create message
message = create_purge_recipe_message(recipe_id=recipe.id,
trigger_id=trigger.id,
source_file_id=file_2.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Check results are accurate
self.assertEqual(
PurgeResults.objects.values_list(
'num_recipes_deleted',
flat=True).get(source_file_id=file_2.id), 0)
def test_json(self):
"""Tests coverting a PurgeRecipe message to and from JSON"""
# Create message
message = create_purge_recipe_message(recipe_id=self.recipe_1.id,
trigger_id=self.trigger.id,
source_file_id=self.file_1.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = PurgeRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
# Check to see if a force stop was placed on this purge process
results = PurgeResults.objects.get(trigger_event=self.trigger_id)
if results.force_stop_purge:
return True
# Kick off purge_source_file for the source file input
self.new_messages.append(
create_purge_source_file_message(
source_file_id=self.source_file_id,
trigger_id=self.trigger_id))
# Kick off purge_recipe for recipe with node job
parent_recipes = RecipeNode.objects.filter(job__in=self._purge_job_ids,
is_original=True)
for recipe_node in parent_recipes:
self.new_messages.append(
create_purge_recipe_message(
recipe_id=recipe_node.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
with transaction.atomic():
job_exe_queryset = JobExecution.objects.filter(
job__in=self._purge_job_ids)
TaskUpdate.objects.filter(job_exe__in=job_exe_queryset).delete()
JobExecutionOutput.objects.filter(
job_exe__in=job_exe_queryset).delete()
JobExecutionEnd.objects.filter(
job_exe__in=job_exe_queryset).delete()
FileAncestryLink.objects.filter(
job__in=self._purge_job_ids).delete()
job_exe_queryset.delete()
BatchJob.objects.filter(job__in=self._purge_job_ids).delete()
RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
Queue.objects.filter(job__in=self._purge_job_ids).delete()
Job.objects.filter(id__in=self._purge_job_ids).delete()
# Update results
results.num_jobs_deleted = F('num_jobs_deleted') + len(
self._purge_job_ids)
results.save()
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
# Check to see if a force stop was placed on this purge process
results = PurgeResults.objects.get(trigger_event=self.trigger_id)
if results.force_stop_purge:
return True
job_inputs = JobInputFile.objects.filter(
input_file=self.source_file_id,
job__recipe__isnull=True).select_related('job')
recipe_inputs = RecipeInputFile.objects.filter(
input_file=self.source_file_id,
recipe__is_superseded=False).select_related('recipe')
# Kick off spawn_delete_job_files for jobs that are not in a recipe and have the given source_file as input
for job_input in job_inputs:
from job.messages.spawn_delete_files_job import create_spawn_delete_files_job
self.new_messages.append(
create_spawn_delete_files_job(
job_id=job_input.job.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id,
purge=True))
# Kick off purge_recipe for recipes that are not superseded and have the given source_file as input
for recipe_input in recipe_inputs:
from recipe.messages.purge_recipe import create_purge_recipe_message
self.new_messages.append(
create_purge_recipe_message(
recipe_id=recipe_input.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
# Delete Ingest and ScaleFile
if not job_inputs and not recipe_inputs:
Ingest.objects.filter(source_file=self.source_file_id).delete()
ScaleFile.objects.filter(id=self.source_file_id).delete()
# Update results
results.purge_completed = timezone.now()
results.save()
return True
def test_execute_no_leaf_nodes(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create recipes
recipe_type = recipe_test_utils.create_recipe_type()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
# Create message
message = create_purge_recipe_message(recipe_id=recipe.id,
trigger_id=self.trigger.id,
source_file_id=self.file_1.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Assert models were deleted
self.assertEqual(Recipe.objects.filter(id=recipe.id).count(), 0)
self.assertEqual(RecipeNode.objects.filter(recipe=recipe).count(), 0)
def test_execute_with_sub_recipe(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create recipes
sub_recipe_type = recipe_test_utils.create_recipe_type()
definition = RecipeDefinition(Interface())
definition.add_recipe_node('A', sub_recipe_type.name,
sub_recipe_type.revision_num)
recipe_a = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_a.jobs_completed = 3
recipe_a.jobs_running = 2
recipe_a.jobs_total = 5
Recipe.objects.bulk_create([recipe_a])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='A', sub_recipe=recipe_a, save=False)
RecipeNode.objects.bulk_create([recipe_node_a])
# Create message
message = create_purge_recipe_message(recipe_id=recipe.id,
trigger_id=self.trigger.id,
source_file_id=self.file_1.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the parent recipe was sent
msgs = [
msg for msg in message.new_messages if msg.type == 'purge_recipe'
]
self.assertEqual(len(msgs), 1)
for msg in msgs:
self.assertEqual(msg.recipe_id, recipe_node_a.sub_recipe.id)
def test_execute_with_jobs(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create message
message = create_purge_recipe_message(recipe_id=self.recipe_1.id,
trigger_id=self.trigger.id,
source_file_id=self.file_1.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that the two jobs in this recipe were called to be purged
msgs = [
msg for msg in message.new_messages
if msg.type == 'spawn_delete_files_job'
]
self.assertEqual(len(msgs), 2)
for msg in msgs:
self.assertIn(msg.job_id, [self.job_1_1.id, self.job_1_2.id])
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
job_inputs = JobInputFile.objects.filter(
input_file=self.source_file_id,
job__recipe__isnull=True).select_related('job')
recipe_inputs = RecipeInputFile.objects.filter(
input_file=self.source_file_id,
recipe__is_superseded=False).select_related('recipe')
# Kick off spawn_delete_job_files for jobs that are not in a recipe and have the given source_file as input
for job_input in job_inputs:
from job.messages.spawn_delete_files_job import create_spawn_delete_files_job
self.new_messages.append(
create_spawn_delete_files_job(
job_id=job_input.job.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id,
purge=True))
# Kick off purge_recipe for recipes that are not superseded and have the given source_file as input
for recipe_input in recipe_inputs:
from recipe.messages.purge_recipe import create_purge_recipe_message
self.new_messages.append(
create_purge_recipe_message(
recipe_id=recipe_input.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
# Delete Ingest and ScaleFile
if not job_inputs and not recipe_inputs:
Ingest.objects.filter(source_file=self.source_file_id).delete()
ScaleFile.objects.filter(id=self.source_file_id).delete()
return True