Esempio n. 1
0
    def test_execute_with_parent_recipe(self):
        """Tests calling PurgeRecipe.execute() successfully"""

        # Create recipes
        recipe = recipe_test_utils.create_recipe()
        parent_recipe = recipe_test_utils.create_recipe()
        recipe_test_utils.create_recipe_node(recipe=parent_recipe,
                                             node_name='A',
                                             sub_recipe=recipe,
                                             save=True)

        # Create message
        message = create_purge_recipe_message(recipe_id=recipe.id,
                                              trigger_id=self.trigger.id,
                                              source_file_id=self.file_1.id)

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        # Test to see that a message to purge the parent recipe was sent
        msgs = [
            msg for msg in message.new_messages if msg.type == 'purge_recipe'
        ]
        self.assertEqual(len(msgs), 1)
        for msg in msgs:
            self.assertEqual(msg.recipe_id, parent_recipe.id)

        # Assert models were deleted
        self.assertEqual(Recipe.objects.filter(id=recipe.id).count(), 0)
        self.assertEqual(RecipeNode.objects.filter(recipe=recipe).count(), 0)
Esempio n. 2
0
    def execute(self):
        """See :meth:`messaging.messages.message.CommandMessage.execute`
        """

        # Kick off purge_source_file for the source file input
        self.new_messages.append(create_purge_source_file_message(source_file_id=self.source_file_id,
                                                                  trigger_id=self.trigger_id))

        # Kick off purge_recipe for recipe with node job
        parent_recipes = RecipeNode.objects.filter(job__in=self._purge_job_ids, is_original=True)
        for recipe_node in parent_recipes:
            self.new_messages.append(create_purge_recipe_message(recipe_id=recipe_node.recipe.id,
                                                                 trigger_id=self.trigger_id,
                                                                 source_file_id=self.source_file_id))

        with transaction.atomic():
            job_exe_queryset = JobExecution.objects.filter(job__in=self._purge_job_ids)
            TaskUpdate.objects.filter(job_exe__in=job_exe_queryset).delete()
            JobExecutionOutput.objects.filter(job_exe__in=job_exe_queryset).delete()
            JobExecutionEnd.objects.filter(job_exe__in=job_exe_queryset).delete()
            job_exe_queryset.delete()
            FileAncestryLink.objects.filter(job__in=self._purge_job_ids).delete()
            BatchJob.objects.filter(job__in=self._purge_job_ids).delete()
            RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
            JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
            Queue.objects.filter(job__in=self._purge_job_ids).delete()
            Job.objects.filter(id__in=self._purge_job_ids).delete()

        return True
Esempio n. 3
0
    def test_execute_force_stop(self):
        """Tests calling PurgeRecipe.execute() successfully"""

        # Create PurgeResults entry
        file_2 = storage_test_utils.create_file(file_type='SOURCE')
        trigger = trigger_test_utils.create_trigger_event()
        PurgeResults.objects.create(source_file_id=file_2.id,
                                    trigger_event=trigger,
                                    force_stop_purge=True)
        self.assertEqual(
            PurgeResults.objects.values_list(
                'num_recipes_deleted',
                flat=True).get(trigger_event=trigger.id), 0)

        # Create recipes
        recipe_type = recipe_test_utils.create_recipe_type()
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)

        # Create message
        message = create_purge_recipe_message(recipe_id=recipe.id,
                                              trigger_id=trigger.id,
                                              source_file_id=file_2.id)

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        # Check results are accurate
        self.assertEqual(
            PurgeResults.objects.values_list(
                'num_recipes_deleted',
                flat=True).get(source_file_id=file_2.id), 0)
Esempio n. 4
0
    def test_json(self):
        """Tests coverting a PurgeRecipe message to and from JSON"""

        # Create message
        message = create_purge_recipe_message(recipe_id=self.recipe_1.id,
                                              trigger_id=self.trigger.id,
                                              source_file_id=self.file_1.id)

        # Convert message to JSON and back, and then execute
        message_json_dict = message.to_json()
        new_message = PurgeRecipe.from_json(message_json_dict)
        result = new_message.execute()

        self.assertTrue(result)
Esempio n. 5
0
    def execute(self):
        """See :meth:`messaging.messages.message.CommandMessage.execute`
        """

        # Check to see if a force stop was placed on this purge process
        results = PurgeResults.objects.get(trigger_event=self.trigger_id)
        if results.force_stop_purge:
            return True

        # Kick off purge_source_file for the source file input
        self.new_messages.append(
            create_purge_source_file_message(
                source_file_id=self.source_file_id,
                trigger_id=self.trigger_id))

        # Kick off purge_recipe for recipe with node job
        parent_recipes = RecipeNode.objects.filter(job__in=self._purge_job_ids,
                                                   is_original=True)
        for recipe_node in parent_recipes:
            self.new_messages.append(
                create_purge_recipe_message(
                    recipe_id=recipe_node.recipe.id,
                    trigger_id=self.trigger_id,
                    source_file_id=self.source_file_id))

        with transaction.atomic():
            job_exe_queryset = JobExecution.objects.filter(
                job__in=self._purge_job_ids)
            TaskUpdate.objects.filter(job_exe__in=job_exe_queryset).delete()
            JobExecutionOutput.objects.filter(
                job_exe__in=job_exe_queryset).delete()
            JobExecutionEnd.objects.filter(
                job_exe__in=job_exe_queryset).delete()
            FileAncestryLink.objects.filter(
                job__in=self._purge_job_ids).delete()
            job_exe_queryset.delete()
            BatchJob.objects.filter(job__in=self._purge_job_ids).delete()
            RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
            JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
            Queue.objects.filter(job__in=self._purge_job_ids).delete()
            Job.objects.filter(id__in=self._purge_job_ids).delete()

            # Update results
            results.num_jobs_deleted = F('num_jobs_deleted') + len(
                self._purge_job_ids)
            results.save()

        return True
Esempio n. 6
0
    def execute(self):
        """See :meth:`messaging.messages.message.CommandMessage.execute`
        """

        # Check to see if a force stop was placed on this purge process
        results = PurgeResults.objects.get(trigger_event=self.trigger_id)
        if results.force_stop_purge:
            return True

        job_inputs = JobInputFile.objects.filter(
            input_file=self.source_file_id,
            job__recipe__isnull=True).select_related('job')
        recipe_inputs = RecipeInputFile.objects.filter(
            input_file=self.source_file_id,
            recipe__is_superseded=False).select_related('recipe')

        # Kick off spawn_delete_job_files for jobs that are not in a recipe and have the given source_file as input
        for job_input in job_inputs:
            from job.messages.spawn_delete_files_job import create_spawn_delete_files_job
            self.new_messages.append(
                create_spawn_delete_files_job(
                    job_id=job_input.job.id,
                    trigger_id=self.trigger_id,
                    source_file_id=self.source_file_id,
                    purge=True))

        # Kick off purge_recipe for recipes that are not superseded and have the given source_file as input
        for recipe_input in recipe_inputs:
            from recipe.messages.purge_recipe import create_purge_recipe_message
            self.new_messages.append(
                create_purge_recipe_message(
                    recipe_id=recipe_input.recipe.id,
                    trigger_id=self.trigger_id,
                    source_file_id=self.source_file_id))

        # Delete Ingest and ScaleFile
        if not job_inputs and not recipe_inputs:
            Ingest.objects.filter(source_file=self.source_file_id).delete()
            ScaleFile.objects.filter(id=self.source_file_id).delete()

            # Update results
            results.purge_completed = timezone.now()
            results.save()

        return True
Esempio n. 7
0
    def test_execute_no_leaf_nodes(self):
        """Tests calling PurgeRecipe.execute() successfully"""

        # Create recipes
        recipe_type = recipe_test_utils.create_recipe_type()
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)

        # Create message
        message = create_purge_recipe_message(recipe_id=recipe.id,
                                              trigger_id=self.trigger.id,
                                              source_file_id=self.file_1.id)

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        # Assert models were deleted
        self.assertEqual(Recipe.objects.filter(id=recipe.id).count(), 0)
        self.assertEqual(RecipeNode.objects.filter(recipe=recipe).count(), 0)
Esempio n. 8
0
    def test_execute_with_sub_recipe(self):
        """Tests calling PurgeRecipe.execute() successfully"""

        # Create recipes
        sub_recipe_type = recipe_test_utils.create_recipe_type()

        definition = RecipeDefinition(Interface())
        definition.add_recipe_node('A', sub_recipe_type.name,
                                   sub_recipe_type.revision_num)

        recipe_a = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
                                                   save=False)
        recipe_a.jobs_completed = 3
        recipe_a.jobs_running = 2
        recipe_a.jobs_total = 5
        Recipe.objects.bulk_create([recipe_a])

        definition_json_dict = convert_recipe_definition_to_v6_json(
            definition).get_dict()
        recipe_type = recipe_test_utils.create_recipe_type(
            definition=definition_json_dict)
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)

        recipe_node_a = recipe_test_utils.create_recipe_node(
            recipe=recipe, node_name='A', sub_recipe=recipe_a, save=False)
        RecipeNode.objects.bulk_create([recipe_node_a])

        # Create message
        message = create_purge_recipe_message(recipe_id=recipe.id,
                                              trigger_id=self.trigger.id,
                                              source_file_id=self.file_1.id)

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        # Test to see that a message to purge the parent recipe was sent
        msgs = [
            msg for msg in message.new_messages if msg.type == 'purge_recipe'
        ]
        self.assertEqual(len(msgs), 1)
        for msg in msgs:
            self.assertEqual(msg.recipe_id, recipe_node_a.sub_recipe.id)
Esempio n. 9
0
    def test_execute_with_jobs(self):
        """Tests calling PurgeRecipe.execute() successfully"""

        # Create message
        message = create_purge_recipe_message(recipe_id=self.recipe_1.id,
                                              trigger_id=self.trigger.id,
                                              source_file_id=self.file_1.id)

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        # Test to see that the two jobs in this recipe were called to be purged
        msgs = [
            msg for msg in message.new_messages
            if msg.type == 'spawn_delete_files_job'
        ]
        self.assertEqual(len(msgs), 2)
        for msg in msgs:
            self.assertIn(msg.job_id, [self.job_1_1.id, self.job_1_2.id])
Esempio n. 10
0
    def execute(self):
        """See :meth:`messaging.messages.message.CommandMessage.execute`
        """

        job_inputs = JobInputFile.objects.filter(
            input_file=self.source_file_id,
            job__recipe__isnull=True).select_related('job')
        recipe_inputs = RecipeInputFile.objects.filter(
            input_file=self.source_file_id,
            recipe__is_superseded=False).select_related('recipe')

        # Kick off spawn_delete_job_files for jobs that are not in a recipe and have the given source_file as input
        for job_input in job_inputs:
            from job.messages.spawn_delete_files_job import create_spawn_delete_files_job
            self.new_messages.append(
                create_spawn_delete_files_job(
                    job_id=job_input.job.id,
                    trigger_id=self.trigger_id,
                    source_file_id=self.source_file_id,
                    purge=True))

        # Kick off purge_recipe for recipes that are not superseded and have the given source_file as input
        for recipe_input in recipe_inputs:
            from recipe.messages.purge_recipe import create_purge_recipe_message
            self.new_messages.append(
                create_purge_recipe_message(
                    recipe_id=recipe_input.recipe.id,
                    trigger_id=self.trigger_id,
                    source_file_id=self.source_file_id))

        # Delete Ingest and ScaleFile
        if not job_inputs and not recipe_inputs:
            Ingest.objects.filter(source_file=self.source_file_id).delete()
            ScaleFile.objects.filter(id=self.source_file_id).delete()

        return True