def test_execute_force_stop(self):
"""Tests calling PurgeSourceFile.execute() successfully"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
# Create PurgeResults entry
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger,
force_stop_purge=True)
self.assertIsNone(
PurgeResults.objects.values_list(
'purge_completed', flat=True).get(trigger_event=trigger))
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=trigger.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that the PurgeResults was completed
self.assertIsNone(
PurgeResults.objects.values_list(
'purge_completed', flat=True).get(trigger_event=trigger))
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
# Kick off purge_source_file for the source file input
self.new_messages.append(create_purge_source_file_message(source_file_id=self.source_file_id,
trigger_id=self.trigger_id))
# Kick off purge_recipe for recipe with node job
parent_recipes = RecipeNode.objects.filter(job__in=self._purge_job_ids, is_original=True)
for recipe_node in parent_recipes:
self.new_messages.append(create_purge_recipe_message(recipe_id=recipe_node.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
with transaction.atomic():
job_exe_queryset = JobExecution.objects.filter(job__in=self._purge_job_ids)
TaskUpdate.objects.filter(job_exe__in=job_exe_queryset).delete()
JobExecutionOutput.objects.filter(job_exe__in=job_exe_queryset).delete()
JobExecutionEnd.objects.filter(job_exe__in=job_exe_queryset).delete()
job_exe_queryset.delete()
FileAncestryLink.objects.filter(job__in=self._purge_job_ids).delete()
BatchJob.objects.filter(job__in=self._purge_job_ids).delete()
RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
Queue.objects.filter(job__in=self._purge_job_ids).delete()
Job.objects.filter(id__in=self._purge_job_ids).delete()
return True
def test_execute_with_recipe(self):
"""Tests calling PurgeSourceFile.execute() successfully"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger)
# Create a recipe and other models
recipe = recipe_test_utils.create_recipe()
recipe_test_utils.create_input_file(recipe=recipe,
input_file=source_file)
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=trigger.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the recipe was created
self.assertEqual(len(message.new_messages), 1)
for msg in message.new_messages:
self.assertEqual(msg.recipe_id, recipe.id)
self.assertEqual(msg.type, 'purge_recipe')
def test_execute_with_job(self):
"""Tests calling PurgeSourceFile.execute() successfully"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger)
# Create a job and other models
job = job_test_utils.create_job()
job_test_utils.create_input_file(job=job, input_file=source_file)
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=trigger.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the job was created
self.assertEqual(len(message.new_messages), 1)
for msg in message.new_messages:
self.assertEqual(msg.job_id, job.id)
self.assertEqual(msg.type, 'spawn_delete_files_job')
def test_json(self):
"""Tests coverting a PurgeSourceFile message to and from JSON"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=self.trigger.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = PurgeSourceFile.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
# Check to see if a force stop was placed on this purge process
results = PurgeResults.objects.get(trigger_event=self.trigger_id)
if results.force_stop_purge:
return True
# Kick off purge_source_file for the source file input
self.new_messages.append(
create_purge_source_file_message(
source_file_id=self.source_file_id,
trigger_id=self.trigger_id))
# Kick off purge_recipe for recipe with node job
parent_recipes = RecipeNode.objects.filter(job__in=self._purge_job_ids,
is_original=True)
for recipe_node in parent_recipes:
self.new_messages.append(
create_purge_recipe_message(
recipe_id=recipe_node.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
with transaction.atomic():
job_exe_queryset = JobExecution.objects.filter(
job__in=self._purge_job_ids)
TaskUpdate.objects.filter(job_exe__in=job_exe_queryset).delete()
JobExecutionOutput.objects.filter(
job_exe__in=job_exe_queryset).delete()
JobExecutionEnd.objects.filter(
job_exe__in=job_exe_queryset).delete()
FileAncestryLink.objects.filter(
job__in=self._purge_job_ids).delete()
job_exe_queryset.delete()
BatchJob.objects.filter(job__in=self._purge_job_ids).delete()
RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
Queue.objects.filter(job__in=self._purge_job_ids).delete()
Job.objects.filter(id__in=self._purge_job_ids).delete()
# Update results
results.num_jobs_deleted = F('num_jobs_deleted') + len(
self._purge_job_ids)
results.save()
return True
def post(self, request):
"""Kicks off the process of purging a given source file from Scale
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
if self.request.version != 'v6' and self.request.version != 'v7':
content = 'This endpoint is supported with REST API v6+'
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
file_id = rest_util.parse_int(request, 'file_id')
try:
file_id = int(file_id)
except ValueError:
content = 'The given file_id is not valid: %i' % (file_id)
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
# Attempt to fetch the ScaleFile model
try:
source_file = ScaleFile.objects.get(id=file_id)
except ScaleFile.DoesNotExist:
content = 'No file record exists for the given file_id: %i' % (
file_id)
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
# Inspect the file to ensure it will purge correctly
if source_file.file_type != 'SOURCE':
content = 'The given file_id does not correspond to a SOURCE file_type: %i' % (
file_id)
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
event = TriggerEvent.objects.create_trigger_event(
'USER', None, {'user': '******'}, now())
PurgeResults.objects.create(source_file_id=file_id,
trigger_event=event)
CommandMessageManager().send_messages([
create_purge_source_file_message(source_file_id=file_id,
trigger_id=event.id)
])
return Response(status=status.HTTP_204_NO_CONTENT)
def test_execute(self):
"""Tests calling PurgeSourceFile.execute() successfully"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=self.trigger.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that the ScaleFile and Ingest records were deleted
self.assertEqual(
ScaleFile.objects.filter(id=source_file.id).count(), 0)
self.assertEqual(
Ingest.objects.filter(source_file=source_file.id).count(), 0)
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
# Check to see if a force stop was placed on this purge process
results = PurgeResults.objects.get(trigger_event=self.trigger_id)
if results.force_stop_purge:
return True
recipe = Recipe.objects.select_related('superseded_recipe').get(
id=self.recipe_id)
# Kick off purge_source_file for the source file
self.new_messages.append(
create_purge_source_file_message(
source_file_id=self.source_file_id,
trigger_id=self.trigger_id))
recipe_inst = Recipe.objects.get_recipe_instance(self.recipe_id)
recipe_nodes = recipe_inst.get_original_leaf_nodes(
) # {Node_Name: Node}
parent_recipes = RecipeNode.objects.filter(sub_recipe=recipe,
is_original=True)
if recipe_nodes:
# Kick off a delete_files job for leaf node jobs
leaf_jobs = [
node for node in recipe_nodes.values()
if node.node_type == JobNodeDefinition.NODE_TYPE
]
for node in leaf_jobs:
self.new_messages.append(
create_spawn_delete_files_job(
job_id=node.job.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id,
purge=True))
# Kick off a purge_recipe for leaf node recipes
leaf_recipes = [
node for node in recipe_nodes.values()
if node.node_type == RecipeNodeDefinition.NODE_TYPE
]
for node in leaf_recipes:
self.new_messages.append(
create_purge_recipe_message(
recipe_id=node.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
else:
# Kick off a purge_recipe for a parent recipe
if parent_recipes:
for parent_recipe in parent_recipes:
self.new_messages.append(
create_purge_recipe_message(
recipe_id=parent_recipe.recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
RecipeNode.objects.filter(sub_recipe=recipe).delete()
# Kick off purge_recipe for a superseded recipe
elif recipe.superseded_recipe:
self.new_messages.append(
create_purge_recipe_message(
recipe_id=recipe.superseded_recipe.id,
trigger_id=self.trigger_id,
source_file_id=self.source_file_id))
# Delete RecipeNode, RecipeInputFile, and Recipe
RecipeNode.objects.filter(Q(recipe=recipe)
| Q(sub_recipe=recipe)).delete()
RecipeInputFile.objects.filter(recipe=recipe).delete()
recipe.delete()
# Update results
PurgeResults.objects.filter(trigger_event=self.trigger_id).update(
num_recipes_deleted=F('num_recipes_deleted') + 1)
return True
