Exemplo n.º 1
0
    def test_json(self):
        """Tests converting a ProcessJobInput message to and from JSON"""

        job = job_test_utils.create_job(num_exes=0, status='PENDING', input_file_size=None, input=DataV6().get_dict())

        # Create message
        message = ProcessJobInput()
        message.job_id = job.id

        # Convert message to JSON and back, and then execute
        message_json_dict = message.to_json()
        new_message = ProcessJobInput.from_json(message_json_dict)
        result = new_message.execute()

        self.assertTrue(result)
        job = Job.objects.get(id=job.id)
        self.assertEqual(len(new_message.new_messages), 1)
        self.assertEqual(new_message.new_messages[0].type, 'queued_jobs')
        self.assertFalse(new_message.new_messages[0].requeue)
        # Job should have input_file_size set to 0 (no input files)
        self.assertEqual(job.input_file_size, 0.0)
Exemplo n.º 2
0
    def test_execute_with_data(self):
        """Tests calling ProcessJobInput.execute() successfully when the job already has data populated"""

        workspace = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0)
        file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0)

        inputs = [{
                    'name': 'Input 1',
                    'mediaTypes': ['text/plain'],
                }, {
                    'name': 'Input 2',
                    'mediaTypes': ['text/plain'],
                }]
        outputs = [{
                    'name': 'Output 1',
                    'mediaType': 'image/png',
            }]
        manifest = job_test_utils.create_seed_manifest(command='my_command', inputs_files=inputs, outputs_files=outputs)
        job_type = job_test_utils.create_seed_job_type(manifest=manifest)

        input_dict = {
            'version': '1.0',
            'input_data': [{
                'name': 'Input 1',
                'file_id': file_1.id
            }, {
                'name': 'Input 2',
                'file_id': file_2.id
            }],
            'output_data': [{
                'name': 'Output 1',
                'workspace_id': workspace.id
            }]}

        job = job_test_utils.create_job(job_type=job_type, num_exes=0, status='PENDING', input_file_size=None,
                                        input=input_dict)

        # Create message
        message = ProcessJobInput()
        message.job_id = job.id

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        job = Job.objects.get(id=job.id)
        # Check for queued jobs message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'queued_jobs')
        self.assertFalse(message.new_messages[0].requeue)

        # Check job for expected input_file_size
        self.assertEqual(job.input_file_size, 1042.0)

        # Make sure job input file models are created
        job_input_files = JobInputFile.objects.filter(job_id=job.id)
        self.assertEqual(len(job_input_files), 2)
        for job_input_file in job_input_files:
            if job_input_file.job_input == 'Input 1':
                self.assertEqual(job_input_file.input_file_id, file_1.id)
            elif job_input_file.job_input == 'Input 2':
                self.assertEqual(job_input_file.input_file_id, file_2.id)
            else:
                self.fail('Invalid input name: %s' % job_input_file.job_input)

        # Test executing message again
        message_json_dict = message.to_json()
        message = ProcessJobInput.from_json(message_json_dict)
        result = message.execute()
        self.assertTrue(result)

        # Still should have queued jobs message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'queued_jobs')
        self.assertFalse(message.new_messages[0].requeue)

        # Make sure job input file models are unchanged
        job_input_files = JobInputFile.objects.filter(job_id=job.id)
        self.assertEqual(len(job_input_files), 2)
Exemplo n.º 3
0
    def test_execute_with_recipe_missing_output(self):
        """Tests calling ProcessJobInput.execute() when a job has to get its data from its recipe but the previous job didn't provide it's output"""

        workspace = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0)
        file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0)
        file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0)
        file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0)
        manifest_1 = {
            'seedVersion': '1.0.0',
            'job': {
                'name': 'job-a',
                'jobVersion': '1.0.0',
                'packageVersion': '1.0.0',
                'title': '',
                'description': '',
                'maintainer': {
                    'name': 'John Doe',
                    'email': '*****@*****.**'
                },
                'timeout': 10,
                'interface': {
                    'command': '',
                    'inputs': {'files': [], 'json': []},
                    'outputs': {
                        'files': [{'name': 'OUTPUT_A', 'pattern': '*.png'}]
                    }
                }
            }
        }
        job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1)
        manifest_2 = {
            'seedVersion': '1.0.0',
            'job': {
                'name': 'job-b',
                'jobVersion': '1.0.0',
                'packageVersion': '1.0.0',
                'title': '',
                'description': '',
                'maintainer': {
                    'name': 'John Doe',
                    'email': '*****@*****.**'
                },
                'timeout': 10,
                'interface': {
                    'command': '',
                    'inputs': {'files': [{'name': 'INPUT_B', 'multiple': True}]},
                    'outputs': {
                        'files': [{'name': 'OUTPUT_B', 'pattern': '*.png'}]
                    }
                }
            }
        }
        job_type_2 = job_test_utils.create_seed_job_type(manifest=manifest_2)
        output_dict = {
            'version': '1.0',
            'output_data': [{
            }]
        }
        job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict)
        job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None,
                                          input=None)

        from recipe.definition.definition import RecipeDefinition
        from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json
        from recipe.test import utils as recipe_test_utils
        definition = RecipeDefinition(Interface())
        definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num)
        definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num)
        definition.add_dependency('node_a', 'node_b')
        definition.add_dependency_input_connection('node_b', 'INPUT_B', 'node_a', 'OUTPUT_A')
        def_dict = convert_recipe_definition_to_v1_json(definition).get_dict()
        recipe_type = recipe_test_utils.create_recipe_type_v6(definition=def_dict)
        recipe_data_dict = {'version': '1.0', 'input_data': [], 'workspace_id': workspace.id}
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict)
        recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1)
        recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2)
        job_1.recipe = recipe
        job_1.save()
        job_2.recipe = recipe
        job_2.save()

        # Create message
        message = ProcessJobInput()
        message.job_id = job_2.id

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        job_2 = Job.objects.get(id=job_2.id)
        # Check for queued jobs message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'cancel_jobs')
Exemplo n.º 4
0
    def test_execute_with_recipe(self):
        """Tests calling ProcessJobInput.execute() successfully when a job has to get its data from its recipe"""

        workspace = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0)
        file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0)
        file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0)
        file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0)
        manifest_1 = {
            'seedVersion': '1.0.0',
            'job': {
                'name': 'job-a',
                'jobVersion': '1.0.0',
                'packageVersion': '1.0.0',
                'title': '',
                'description': '',
                'maintainer': {
                    'name': 'John Doe',
                    'email': '*****@*****.**'
                },
                'timeout': 10,
                'interface': {
                    'command': '',
                    'inputs': {'files': [], 'json': []},
                    'outputs': {
                        'files': [{'name': 'OUTPUT_A', 'pattern': '*.png'}]
                    }
                }
            }
        }
        job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1)
        manifest_2 = {
            'seedVersion': '1.0.0',
            'job': {
                'name': 'job-b',
                'jobVersion': '1.0.0',
                'packageVersion': '1.0.0',
                'title': '',
                'description': '',
                'maintainer': {
                    'name': 'John Doe',
                    'email': '*****@*****.**'
                },
                'timeout': 10,
                'interface': {
                    'command': '',
                    'inputs': {'files': [{'name': 'INPUT_B', 'multiple': True}]},
                    'outputs': {
                        'files': [{'name': 'OUTPUT_B', 'pattern': '*.png'}]
                    }
                }
            }
        }
        job_type_2 = job_test_utils.create_seed_job_type(manifest=manifest_2)
        output_dict = {
            'version': '1.0',
            'output_data': [{
                'name': 'OUTPUT_A',
                'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id]
            }]
        }
        job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict)
        job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None,
                                          input=None)

        from recipe.definition.definition import RecipeDefinition
        from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json
        from recipe.test import utils as recipe_test_utils
        definition = RecipeDefinition(Interface())
        definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num)
        definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num)
        definition.add_dependency('node_a', 'node_b')
        definition.add_dependency_input_connection('node_b', 'INPUT_B', 'node_a', 'OUTPUT_A')
        def_dict = convert_recipe_definition_to_v1_json(definition).get_dict()
        recipe_type = recipe_test_utils.create_recipe_type_v6(definition=def_dict)
        recipe_data_dict = {'version': '1.0', 'input_data': [], 'workspace_id': workspace.id}
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict)
        recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1)
        recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2)
        job_1.recipe = recipe
        job_1.save()
        job_2.recipe = recipe
        job_2.save()

        # Create message
        message = ProcessJobInput()
        message.job_id = job_2.id

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        job_2 = Job.objects.get(id=job_2.id)
        # Check for queued jobs message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'queued_jobs')
        self.assertFalse(message.new_messages[0].requeue)

        # Check job for expected input_file_size
        self.assertEqual(job_2.input_file_size, 24469.0)
        # Check job for expected input data
        self.assertSetEqual(set(job_2.get_input_data().values.keys()), {'INPUT_B'})
        self.assertListEqual(job_2.get_input_data().values['INPUT_B'].file_ids,
                             [file_1.id, file_2.id, file_3.id, file_4.id])

        # Make sure job input file models are created
        job_input_files = JobInputFile.objects.filter(job_id=job_2.id)
        self.assertEqual(len(job_input_files), 4)
        file_ids = set()
        for job_input_file in job_input_files:
            self.assertEqual(job_input_file.job_input, 'INPUT_B')
            file_ids.add(job_input_file.input_file_id)
        self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id})

        # Test executing message again
        message_json_dict = message.to_json()
        message = ProcessJobInput.from_json(message_json_dict)
        result = message.execute()
        self.assertTrue(result)

        # Still should have queued jobs message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'queued_jobs')
        self.assertFalse(message.new_messages[0].requeue)

        # Make sure job input file models are unchanged
        job_input_files = JobInputFile.objects.filter(job_id=job_2.id)
        self.assertEqual(len(job_input_files), 4)
Exemplo n.º 5
0
    def test_execute_with_recipe_legacy(self):
        """Tests calling ProcessJobInput.execute() successfully when a legacy job has to get its data from its recipe"""

        workspace = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=104857600.0)
        file_2 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=987654321.0)
        file_3 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=65456.0)
        file_4 = storage_test_utils.create_file(workspace=workspace,
                                                file_size=24564165456.0)
        interface_1 = {
            'version':
            '1.0',
            'command':
            'my_command',
            'command_arguments':
            'args',
            'input_data': [],
            'output_data': [{
                'name': 'Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]
        }
        job_type_1 = job_test_utils.create_job_type(interface=interface_1)
        interface_2 = {
            'version':
            '1.0',
            'command':
            'my_command',
            'command_arguments':
            'args',
            'input_data': [{
                'name': 'Input 1',
                'type': 'files',
                'media_types': ['image/png'],
            }],
            'output_data': [{
                'name': 'New Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]
        }
        job_type_2 = job_test_utils.create_job_type(interface=interface_2)
        output_dict = {
            'version':
            '1.0',
            'output_data': [{
                'name':
                'Output 1',
                'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id]
            }]
        }
        job_1 = job_test_utils.create_job(job_type=job_type_1,
                                          num_exes=1,
                                          status='COMPLETED',
                                          output=output_dict)
        job_2 = job_test_utils.create_job(job_type=job_type_2,
                                          num_exes=0,
                                          status='PENDING',
                                          input_file_size=None,
                                          input=None)

        from recipe.definition.definition import RecipeDefinition
        from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json
        from recipe.test import utils as recipe_test_utils
        definition = RecipeDefinition(Interface())
        definition.add_job_node('node_a', job_type_1.name, job_type_1.version,
                                job_type_1.revision_num)
        definition.add_job_node('node_b', job_type_2.name, job_type_2.version,
                                job_type_2.revision_num)
        definition.add_dependency('node_b', 'node_a')
        definition.add_dependency_input_connection('node_b', 'Input 1',
                                                   'node_a', 'Output 1')
        def_dict = convert_recipe_definition_to_v1_json(definition).get_dict()
        recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict)
        recipe_data_dict = {
            'version': '1.0',
            'input_data': [],
            'workspace_id': workspace.id
        }
        recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
                                                 input=recipe_data_dict)
        recipe_test_utils.create_recipe_job(recipe=recipe,
                                            job_name='node_a',
                                            job=job_1)
        recipe_test_utils.create_recipe_job(recipe=recipe,
                                            job_name='node_b',
                                            job=job_2)
        job_1.recipe = recipe
        job_1.save()
        job_2.recipe = recipe
        job_2.save()

        # Create message
        message = ProcessJobInput()
        message.job_id = job_2.id

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        job_2 = Job.objects.get(id=job_2.id)
        # Check for queued jobs message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'queued_jobs')
        self.assertFalse(message.new_messages[0].requeue)

        # Check job for expected input_file_size
        self.assertEqual(job_2.input_file_size, 24469.0)
        # Check job for expected output workspaces in job input data (legacy)
        self.assertDictEqual(
            job_2.input, {
                'version':
                '1.0',
                'input_data': [{
                    'name':
                    'Input 1',
                    'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id]
                }],
                'output_data': [{
                    'name': 'New Output 1',
                    'workspace_id': workspace.id
                }]
            })

        # Make sure job input file models are created
        job_input_files = JobInputFile.objects.filter(job_id=job_2.id)
        self.assertEqual(len(job_input_files), 4)
        file_ids = set()
        for job_input_file in job_input_files:
            self.assertEqual(job_input_file.job_input, 'Input 1')
            file_ids.add(job_input_file.input_file_id)
        self.assertSetEqual(file_ids,
                            {file_1.id, file_2.id, file_3.id, file_4.id})

        # Test executing message again
        message_json_dict = message.to_json()
        message = ProcessJobInput.from_json(message_json_dict)
        result = message.execute()
        self.assertTrue(result)

        # Still should have queued jobs message
        self.assertEqual(len(message.new_messages), 1)
        self.assertEqual(message.new_messages[0].type, 'queued_jobs')
        self.assertFalse(message.new_messages[0].requeue)

        # Make sure job input file models are unchanged
        job_input_files = JobInputFile.objects.filter(job_id=job_2.id)
        self.assertEqual(len(job_input_files), 4)