Example #1
0
    def test_json_forced_nodes(self):
        """Tests coverting a ProcessRecipeInput message to and from JSON with forced nodes provided"""

        data_dict = convert_data_to_v6_json(Data()).get_dict()
        recipe = recipe_test_utils.create_recipe(input=data_dict)
        forced_nodes = ForcedNodes()
        forced_nodes.set_all_nodes()
        forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()

        # Create message
        message = create_process_recipe_input_messages(
            [recipe.id], forced_nodes=forced_nodes)[0]

        # Convert message to JSON and back, and then execute
        message_json_dict = message.to_json()
        new_message = ProcessRecipeInput.from_json(message_json_dict)
        result = new_message.execute()

        self.assertTrue(result)
        recipe = Recipe.objects.get(id=recipe.id)
        self.assertEqual(len(new_message.new_messages), 1)
        msg = new_message.new_messages[0]
        self.assertEqual(msg.type, 'update_recipe')
        self.assertEqual(msg.root_recipe_id, recipe.id)
        self.assertDictEqual(
            convert_forced_nodes_to_v6(msg.forced_nodes).get_dict(),
            forced_nodes_dict)
        # Recipe should have input_file_size set to 0 (no input files)
        self.assertEqual(recipe.input_file_size, 0.0)
Example #2
0
    def test_json(self):
        """Tests coverting a ProcessRecipeInput message to and from JSON"""

        data_dict = convert_data_to_v6_json(Data()).get_dict()
        recipe = recipe_test_utils.create_recipe(input=data_dict)

        # Create message
        message = ProcessRecipeInput()
        message.recipe_id = recipe.id

        # Convert message to JSON and back, and then execute
        message_json_dict = message.to_json()
        new_message = ProcessRecipeInput.from_json(message_json_dict)
        result = new_message.execute()

        self.assertTrue(result)
        recipe = Recipe.objects.get(id=recipe.id)
        self.assertEqual(len(new_message.new_messages), 1)
        self.assertEqual(new_message.new_messages[0].type, 'update_recipes')
        # Recipe should have input_file_size set to 0 (no input files)
        self.assertEqual(recipe.input_file_size, 0.0)
Example #3
0
    def test_execute_with_data(self):
        """Tests calling ProcessRecipeInput.execute() successfully when the recipe already has data populated"""

        workspace = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=10485760.0)
        file_2 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=104857600.0)
        file_3 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=987654321.0)
        recipe_interface = Interface()
        recipe_interface.add_parameter(FileParameter('input_a',
                                                     ['text/plain']))
        recipe_interface.add_parameter(
            FileParameter('input_b', ['text/plain'], multiple=True))
        definition = RecipeDefinition(recipe_interface)
        definition_dict = convert_recipe_definition_to_v6_json(
            definition).get_dict()
        recipe_type = recipe_test_utils.create_recipe_type(
            definition=definition_dict)

        data = Data()
        data.add_value(FileValue('input_a', [file_1.id]))
        data.add_value(FileValue('input_b', [file_2.id, file_3.id]))
        data_dict = convert_data_to_v6_json(data).get_dict()
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
                                                 input=data_dict)

        # Create message
        message = ProcessRecipeInput()
        message.recipe_id = recipe.id

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        recipe = Recipe.objects.get(id=recipe.id)
        # Check for update_recipes message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'update_recipes')

        # Check recipe for expected input_file_size
        self.assertEqual(recipe.input_file_size, 1052.0)

        # Make sure recipe input file models are created
        recipe_input_files = RecipeInputFile.objects.filter(
            recipe_id=recipe.id)
        self.assertEqual(len(recipe_input_files), 3)
        for recipe_input_file in recipe_input_files:
            if recipe_input_file.input_file_id == file_1.id:
                self.assertEqual(recipe_input_file.recipe_input, 'input_a')
            elif recipe_input_file.input_file_id == file_2.id:
                self.assertEqual(recipe_input_file.recipe_input, 'input_b')
            elif recipe_input_file.input_file_id == file_3.id:
                self.assertEqual(recipe_input_file.recipe_input, 'input_b')
            else:
                self.fail('Invalid input file ID: %s' %
                          recipe_input_file.input_file_id)

        # Test executing message again
        message_json_dict = message.to_json()
        message = ProcessRecipeInput.from_json(message_json_dict)
        result = message.execute()
        self.assertTrue(result)

        # Still should have update_recipes message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'update_recipes')

        # Make sure recipe input file models are unchanged
        recipe_input_files = RecipeInputFile.objects.filter(
            recipe_id=recipe.id)
        self.assertEqual(len(recipe_input_files), 3)
Example #4
0
    def test_execute_with_recipe_legacy(self):
        """Tests calling ProcessRecipeInput.execute() successfully when a legacy sub-recipe has to get its data from its
        recipe
        """

        workspace = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=104857600.0)
        file_2 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=987654321.0)
        file_3 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=65456.0)
        file_4 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=24564165456.0)
        manifest_a = {
            'seedVersion': '1.0.0',
            'job': {
                'name': 'job-a',
                'jobVersion': '1.0.0',
                'packageVersion': '1.0.0',
                'title': '',
                'description': '',
                'maintainer': {
                    'name': 'John Doe',
                    'email': '*****@*****.**'
                },
                'timeout': 10,
                'interface': {
                    'command': '',
                    'inputs': {
                        'files': [],
                        'json': []
                    },
                    'outputs': {
                        'files': [{
                            'name': 'output_a',
                            'pattern': '*.png'
                        }]
                    }
                }
            }
        }
        job_type_a = job_test_utils.create_job_type(interface=manifest_a)
        output_data_a = Data()
        output_data_a.add_value(FileValue('output_a', [file_1.id]))
        output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict()
        manifest_b = {
            'seedVersion': '1.0.0',
            'job': {
                'name': 'job-b',
                'jobVersion': '1.0.0',
                'packageVersion': '1.0.0',
                'title': '',
                'description': '',
                'maintainer': {
                    'name': 'John Doe',
                    'email': '*****@*****.**'
                },
                'timeout': 10,
                'interface': {
                    'command': '',
                    'inputs': {
                        'files': [],
                        'json': []
                    },
                    'outputs': {
                        'files': [{
                            'name': 'output_b',
                            'pattern': '*.png',
                            'multiple': True
                        }]
                    }
                }
            }
        }
        job_type_b = job_test_utils.create_job_type(interface=manifest_b)
        output_data_b = Data()
        output_data_b.add_value(
            FileValue('output_b', [file_2.id, file_3.id, file_4.id]))
        output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict()
        job_a = job_test_utils.create_job(job_type=job_type_a,
                                          num_exes=1,
                                          status='COMPLETED',
                                          output=output_data_a_dict)
        job_b = job_test_utils.create_job(job_type=job_type_b,
                                          num_exes=1,
                                          status='COMPLETED',
                                          output=output_data_b_dict)
        sub_recipe_interface_c = Interface()
        sub_recipe_interface_c.add_parameter(
            FileParameter('input_a', ['image/png']))
        sub_recipe_interface_c.add_parameter(
            FileParameter('input_b', ['image/png'], multiple=True))
        sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c)
        sub_recipe_def_dict_c = convert_recipe_definition_to_v1_json(
            sub_recipe_def_c).get_dict()
        sub_recipe_type_c = recipe_test_utils.create_recipe_type(
            definition=sub_recipe_def_dict_c)
        sub_recipe_c = recipe_test_utils.create_recipe(
            recipe_type=sub_recipe_type_c)

        definition = RecipeDefinition(Interface())
        definition.add_job_node('node_a', job_type_a.name, job_type_a.version,
                                job_type_a.revision_num)
        definition.add_job_node('node_b', job_type_b.name, job_type_b.version,
                                job_type_b.revision_num)
        definition.add_recipe_node('node_c', sub_recipe_type_c.name,
                                   sub_recipe_type_c.revision_num)
        definition.add_dependency('node_c', 'node_a')
        definition.add_dependency_input_connection('node_c', 'input_a',
                                                   'node_a', 'output_a')
        definition.add_dependency('node_c', 'node_b')
        definition.add_dependency_input_connection('node_c', 'input_b',
                                                   'node_b', 'output_b')
        def_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
        recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict)
        recipe_data_dict = {
            'version': '1.0',
            'input_data': [],
            'workspace_id': workspace.id
        }
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
                                                 input=recipe_data_dict)
        recipe_node_a = recipe_test_utils.create_recipe_node(
            recipe=recipe, node_name='node_a', job=job_a)
        recipe_node_b = recipe_test_utils.create_recipe_node(
            recipe=recipe, node_name='node_b', job=job_b)
        recipe_node_c = recipe_test_utils.create_recipe_node(
            recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c)
        RecipeNode.objects.bulk_create(
            [recipe_node_a, recipe_node_b, recipe_node_c])
        job_a.recipe = recipe
        job_a.save()
        job_b.recipe = recipe
        job_b.save()
        sub_recipe_c.recipe = recipe
        sub_recipe_c.save()

        # Create message
        message = ProcessRecipeInput()
        message.recipe_id = sub_recipe_c.id

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id)
        # Check for update_recipes message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'update_recipes')

        # Check sub-recipe for expected input_file_size
        self.assertEqual(sub_recipe_c.input_file_size, 24469.0)
        # Check sub-recipe for expected input data
        self.assertEqual(
            sub_recipe_c.input['version'],
            '1.0')  # Should be legacy input data with workspace ID
        self.assertEqual(sub_recipe_c.input['workspace_id'], workspace.id)
        self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()),
                            {'input_a', 'input_b'})
        self.assertListEqual(
            sub_recipe_c.get_input_data().values['input_a'].file_ids,
            [file_1.id])
        self.assertListEqual(
            sub_recipe_c.get_input_data().values['input_b'].file_ids,
            [file_2.id, file_3.id, file_4.id])

        # Make sure sub-recipe input file models are created
        input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
        self.assertEqual(len(input_files), 4)
        file_ids = {input_file.input_file_id for input_file in input_files}
        self.assertSetEqual(file_ids,
                            {file_1.id, file_2.id, file_3.id, file_4.id})

        # Test executing message again
        message_json_dict = message.to_json()
        message = ProcessRecipeInput.from_json(message_json_dict)
        result = message.execute()
        self.assertTrue(result)

        # Still should have update_recipes message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'update_recipes')

        # Make sure recipe input file models are unchanged
        input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
        self.assertEqual(len(input_files), 4)