Пример #1
0
    def test_input_data_required_must_be_true_or_false(self):
        definition = {
            'command': 'test-command',
            'command_arguments': '${param-1}',
            'version': '1.0',
            'input_data': [{
                'name': 'param-1',
                'type': 'file',
                'required': True,
            }]
        }
        try:
            JobInterface(definition)
        except InvalidInterfaceDefinition:
            self.fail('Valid definition raised a validation exception')

        definition['input_data'][0]['required'] = False
        try:
            JobInterface(definition)
        except InvalidInterfaceDefinition:
            self.fail('Valid definition raised a validation exception')

        definition['input_data'][0]['required'] = 'some_string'
        try:
            JobInterface(definition)
            self.fail('Expected invalid job definition to throw an exception')
        except InvalidInterfaceDefinition:
            pass
Пример #2
0
    def test_no_workspace_needed(self):
        """Tests calling JobInterface.validate_connection() without a workspace, but none is needed."""
        job_interface_dict = {
            'command': 'simple-command',
            'command_arguments': '',
            'version': '1.1',
            'input_data': [{
                'name': 'Input 1',
                'type': 'property',
            }, {
                'name': 'Input 2',
                'type': 'file',
                'media_types': ['text/plain'],
                'partial': True
            }],
            'output_data': [],
        }

        job_interface = JobInterface(job_interface_dict)

        job_conn = JobConnection()
        job_conn.add_property('Input 1')
        job_conn.add_input_file('Input 2', False, ['text/plain'], False, True)

        # No exception is success
        job_interface.validate_connection(job_conn)
Пример #3
0
    def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_isdir):
        def new_retrieve(arg1):
            return {
                'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'],
            }

        mock_retrieve_call.side_effect = new_retrieve
        job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
        job_interface_dict['command_arguments'] = '${files1}'
        job_interface_dict['input_data'] = [{
            'name': 'files1',
            'type': 'files',
            'required': True,
        }]
        job_data_dict['input_data'].append({
            'name': 'files1',
            'file_ids': [1, 2, 3],
        })
        job_data_dict['output_data'].append({
            'name': 'files1_out',
            'workspace_id': self.workspace.id,
        })

        job_interface = JobInterface(job_interface_dict)
        job_data = JobData(job_data_dict)
        job_environment = job_environment_dict
        job_exe_id = 1

        job_interface.perform_pre_steps(job_data, job_environment)
        job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
        expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1')
        self.assertEqual(job_command_arguments, expected_command_arguments,
                         'expected a different command from pre_steps')
Пример #4
0
    def test_output_file(self, mock_loads, mock_open, mock_exists, mock_isfile):
        job_interface_dict, job_data_dict = self._get_simple_interface_data()
        job_interface_dict['output_data'] = [{
            'name': 'output_file',
            'type': 'file',
            'required': True,
        }]
        job_data_dict['output_data'].append({
            'name': 'output_file',
            'workspace_id': self.workspace.id,
        })
        results_manifest = {
            'version': '1.0',
            'files': [{
                'name': 'output_file',
                'path': '/some/path/foo.txt',
            }]
        }
        mock_loads.return_value = results_manifest
        mock_exists.return_value = True
        mock_isfile.return_value = True

        job_exe = MagicMock()

        job_interface = JobInterface(job_interface_dict)
        job_data = Mock(spec=JobData)
        job_data.save_parse_results = Mock()
        fake_stdout = ''

        job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
        job_data.store_output_data_files.assert_called_with({
            'output_file': ('/some/path/foo.txt', None),
        }, job_exe)
Пример #5
0
    def test_successful(self):
        """Tests calling JobInterface.validate_connection() successfully."""
        job_interface_dict = {
            'command': 'simple-command',
            'command_arguments': '',
            'version': '1.0',
            'input_data': [{
                'name': 'Input 1',
                'type': 'property',
            }, {
                'name': 'Input 2',
                'type': 'file',
                'media_types': ['text/plain']
            }],
            'output_data': [{
                'name': 'Output 1',
                'type': 'file',
            }]
        }

        job_interface = JobInterface(job_interface_dict)

        job_conn = JobConnection()
        job_conn.add_property('Input 1')
        job_conn.add_input_file('Input 2', False, ['text/plain'], False, False)
        job_conn.add_workspace()

        # No exception is success
        job_interface.validate_connection(job_conn)
Пример #6
0
    def test_invalid_output_files(self, mock_loads, mock_open, mock_exists, mock_isfile):
        job_interface_dict, job_data_dict = self._get_simple_interface_data()
        job_interface_dict['output_data'] = [{
            'name': 'output_files',
            'type': 'files',
            'required': True,
        }]
        job_data_dict['output_data'].append({
            'name': 'output_files',
            'workspace_id': self.workspace.id,
        })
        results_manifest = {
            'version': '1.0',
            'files': [{
                'name': 'output_files',
                'paths': ['/some/path/foo.txt', '/other/path/foo.txt'],
            }]
        }
        mock_loads.return_value = results_manifest
        mock_exists.return_value = True
        mock_isfile.return_value = False

        job_exe = MagicMock()

        job_interface = JobInterface(job_interface_dict)
        job_data = Mock(spec=JobData)
        job_data.save_parse_results = Mock()
        fake_stdout = ''

        self.assertRaises(InvalidResultsManifest, job_interface.perform_post_steps, job_exe, job_data, fake_stdout)
Пример #7
0
    def test_required_workspace_missing(self):
        """Tests calling JobInterface.validate_connection() when a required workspace is missing"""
        job_interface_dict = {
            'command': 'simple-command',
            'command_arguments': '',
            'version': '1.1',
            'input_data': [{
                'name': 'Input 1',
                'type': 'property',
            }, {
                'name': 'Input 2',
                'type': 'file',
                'media_types': ['text/plain'],
                'partial': True
            }],
            'output_data': [{
                'name': 'Output 1',
                'type': 'file',
            }]
        }

        job_interface = JobInterface(job_interface_dict)

        job_conn = JobConnection()
        job_conn.add_property('Input 1')
        job_conn.add_input_file('Input 2', False, ['text/plain'], False, True)

        self.assertRaises(InvalidConnection, job_interface.validate_connection, job_conn)
Пример #8
0
    def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_isdir):
        job_exe_id = 1

        def new_retrieve(arg1):
            return {
                'file1_out': [input_file_path],
            }

        input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt')
        mock_retrieve_call.side_effect = new_retrieve
        mock_get_one_file.side_effect = lambda (arg1): input_file_path
        job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
        job_interface_dict['command_arguments'] = '${file1}'
        job_interface_dict['input_data'] = [{
            'name': 'file1',
            'type': 'file',
            'required': True,
        }]
        job_data_dict['input_data'].append({
            'name': 'file1',
            'file_id': self.file.id,
        })
        job_data_dict['output_data'].append({
            'name': 'file1_out',
            'workspace_id': self.workspace.id,
        })

        job_interface = JobInterface(job_interface_dict)
        job_data = JobData(job_data_dict)
        job_environment = job_environment_dict

        job_interface.perform_pre_steps(job_data, job_environment)
        job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
        self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
Пример #9
0
    def get_job_interface(self):
        """Returns the interface for this queued job

        :returns: The job interface
        :rtype: :class:`job.configuration.interface.job_interface.JobInterface`
        """

        return JobInterface(self.interface, do_validate=False)
Пример #10
0
def job_type_get_job_interface(self):
    """Returns the interface for running jobs of this type

    :returns: The job interface for this type
    :rtype: :class:`job.configuration.interface.job_interface.JobInterface`
    """

    return JobInterface(self.interface)
Пример #11
0
def job_get_job_interface(self):
    """Returns the interface for this job

    :returns: The interface for this job
    :rtype: :class:`job.configuration.interface.job_interface.JobInterface`
    """

    return JobInterface(self.job_type_rev.interface)
Пример #12
0
def get_job_types(recipe_types=None,
                  job_type_ids=None,
                  job_type_names=None,
                  job_type_categories=None):
    """Exports all the job types in the system based on the given filters.

    :param recipe_types: Only include job types that are referenced by the given recipe types.
    :type recipe_types: list[:class:`recipe.models.RecipeType`]
    :param job_type_ids: A list of unique job type identifiers to include.
    :type job_type_ids: list[str]
    :param job_type_names: A list of job type system names to include.
    :type job_type_names: list[str]
    :param job_type_categories: A list of job type category names to include.
    :type job_type_categories: list[str]
    :returns: A list of matching job types.
    :rtype: list[:class:`job.models.JobType`]
    """

    # Build a set of job type keys referenced by the recipe types
    job_type_keys = set()
    if recipe_types and not (job_type_ids or job_type_names
                             or job_type_categories):
        for recipe_type in recipe_types:
            job_type_keys.update(
                recipe_type.get_recipe_definition().get_job_type_keys())
        if not job_type_keys:
            return []

    # System job types should never be exported
    job_types = JobType.objects.exclude(
        category='system').select_related('trigger_rule')

    # Filter by the referenced job type keys
    job_type_filters = []
    for job_type_key in job_type_keys:
        job_type_filter = Q(name=job_type_key[0], version=job_type_key[1])
        job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter
    if job_type_filters:
        job_types = job_types.filter(job_type_filters)

    # Filter by additional passed arguments
    if job_type_ids:
        job_types = job_types.filter(id__in=job_type_ids)
    if job_type_names:
        job_types = job_types.filter(name__in=job_type_names)
    if job_type_categories:
        job_types = job_types.filter(category__in=job_type_categories)

    # Scrub configuration for secrets
    for job_type in job_types:
        if job_type.configuration:
            configuration = JobConfigurationV2(job_type.configuration)
            interface = JobInterface(job_type.manifest)
            configuration.validate(interface.get_dict())
            job_type.configuration = configuration.get_dict()

    return job_types
Пример #13
0
    def test_output_files_with_geo_metadata(self, mock_loads, mock_open,
                                            mock_exists, mock_isfile):
        job_interface_dict, job_data_dict = self._get_simple_interface_data()
        job_interface_dict['output_data'] = [{
            'name': 'output_files',
            'type': 'files',
            'required': True,
        }]
        job_data_dict['output_data'].append({
            'name': 'output_files',
            'workspace_id': self.workspace.id,
        })
        geo_metadata = {
            'data_started': '2015-05-15T10:34:12Z',
            'data_ended': '2015-05-15T10:36:12Z',
            'geo_json': {
                'type':
                'Polygon',
                'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0],
                                 [1.0, 20.0], [1.0, 10.0]]],
            }
        }
        results_manifest = {
            'version':
            '1.1',
            'output_data': [{
                'name':
                'output_files',
                'files': [{
                    'path': '/some/path/foo.txt',
                    'geo_metadata': geo_metadata,
                }, {
                    'path': '/other/path/foo.txt',
                    'geo_metadata': geo_metadata,
                }]
            }]
        }

        mock_loads.return_value = results_manifest
        mock_exists.return_value = True
        mock_isfile.return_value = True

        job_exe = MagicMock()

        job_interface = JobInterface(job_interface_dict)
        job_data = Mock(spec=JobData)
        job_data.save_parse_results = Mock()
        fake_stdout = ''

        job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
        job_data.store_output_data_files.assert_called_with(
            {
                'output_files': [
                    ('/some/path/foo.txt', None, geo_metadata),
                    ('/other/path/foo.txt', None, geo_metadata),
                ]
            }, job_exe)
Пример #14
0
 def test_interface_must_have_command(self):
     definition = {
         'version': '1.0',
     }
     try:
         JobInterface(definition)
         self.fail('Expected invalid job definition to throw an exception')
     except InvalidInterfaceDefinition:
         pass
Пример #15
0
 def test_minimal_input_validation(self):
     definition = {
         'command': 'test-command',
         'command_arguments': 'some_argument',
         'version': '1.0',
     }
     try:
         JobInterface(definition)
     except InvalidInterfaceDefinition:
         self.fail('A valid definition should not raise an Exception')
Пример #16
0
    def test_bad_input_name(self):
        """Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a bad input name"""

        rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
        rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))

        interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "different_input_name", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
        job_interface = JobInterface(json.loads(interface_json_str))

        self.assertRaises(InvalidConnection, rule_config.validate_trigger_for_job, job_interface)
Пример #17
0
    def test_output_name_appropriate(self):
        good_names = [
            'foo', 'bar', 'baz', 'a file with spaces', 'file_with_underscores'
        ]
        bad_names = [
            'ca$h_money', 'do|do_not', 'try!=found',
            'this_file_is_over_255_characters_long_12345678901234567890123456789012345678901234567890123456789012345678'
            '9012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234'
            '56789012345678901234567890123456789012345678901234567890!'
        ]
        definition = {
            'command': 'test-command',
            'command_arguments': '',
            'version': '1.0',
            'input_data': [{
                'name': 'foo',
                'type': 'file',
                'required': True,
            }],
            'output_data': [{
                'name': 'some_output',
                'type': 'file',
            }]
        }

        for output_name in good_names:
            definition['output_data'][0]['name'] = output_name
            try:
                JobInterface(definition)
            except InvalidInterfaceDefinition:
                self.fail(
                    'Unable to parse a good interface definition with output name: %s'
                    % output_name)
        for output_name in bad_names:
            definition['output_data'][0]['name'] = output_name
            try:
                JobInterface(definition)
                self.fail(
                    'job interface with a bad output name (%s) was able to get past validation'
                    % output_name)
            except InvalidInterfaceDefinition:
                pass
Пример #18
0
    def test_simple_case(self):
        job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()

        job_interface = JobInterface(job_interface_dict)
        job_data = JobData(job_data_dict)
        job_environment = job_environment_dict
        job_exe_id = 1

        job_interface.perform_pre_steps(job_data, job_environment)
        job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
        self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
Пример #19
0
    def test_populate_mounts(self):
        """Tests the addition of mount volumes to the configuration."""

        exe_config = ExecutionConfiguration()

        config_dict = {
            'version': '2.0',
            'mounts': {
                'mount_1': {
                    'type': 'host',
                    'host_path': '/host/path'
                },
                'mount_2': {
                    'type': 'volume',
                    'driver': 'x-driver',
                    'driver_opts': {
                        'foo': 'bar'
                    }
                }
            }
        }

        interface_dict = {
            'version':
            '1.4',
            'command':
            'the cmd',
            'command_arguments':
            'foo',
            'mounts': [{
                'name': 'mount_1',
                'path': '/mount_1',
                'mode': 'ro'
            }, {
                'name': 'mount_2',
                'path': '/mount_2',
                'mode': 'rw'
            }]
        }

        job_exe = MagicMock()
        job_exe.get_job_configuration.return_value = JobConfiguration(
            config_dict)
        job_exe.get_job_interface.return_value = JobInterface(interface_dict)
        job_exe.get_cluster_id.return_value = 'scale_1234'

        exe_config.populate_mounts(job_exe)

        docker_params = exe_config.get_job_task_docker_params()
        self.assertEqual(docker_params[0].flag, 'volume')
        self.assertEqual(docker_params[0].value, '/host/path:/mount_1:ro')
        self.assertEqual(docker_params[1].flag, 'volume')
        mount_2 = '$(docker volume create --name scale_1234_mount_mount_2 --driver x-driver --opt foo=bar):/mount_2:rw'
        self.assertEqual(docker_params[1].value, mount_2)
Пример #20
0
 def test_command_param_will_fail_without_input(self):
     definition = {
         'command': 'test-command',
         'command_arguments': '${param-1}',
         'version': '1.0',
     }
     try:
         JobInterface(definition)
         self.fail('Expected invalid job definition to throw an exception')
     except InvalidInterfaceDefinition:
         pass
Пример #21
0
    def test_manifest_overrides_stdout(self, mock_loads, mock_open,
                                       mock_exists, mock_isfile):
        job_interface_dict, job_data_dict = self._get_simple_interface_data()
        job_interface_dict['input_data'] = [{
            'name': 'input_file',
            'type': 'file',
            'required': True,
        }]
        job_interface_dict['output_data'] = [{
            'name': 'output_file',
            'type': 'file',
            'required': True,
        }, {
            'name': 'output_file_2',
            'type': 'file',
            'required': True,
        }]
        job_data_dict['input_data'].append({
            'name': 'input_file',
            'file_id': self.file.id,
        })
        results_manifest = {
            'version': '1.0',
            'files': [{
                'name': 'output_file',
                'path': '/new/path/foo.txt',
            }]
        }
        mock_loads.return_value = results_manifest
        mock_exists.return_value = True
        mock_isfile.return_value = True

        job_exe = MagicMock()

        job_interface = JobInterface(job_interface_dict)
        job_data = Mock(spec=JobData)
        job_data.save_parse_results = Mock()
        fake_stdout = """
This text is supposed to mimic the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
"""

        job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
        job_data.store_output_data_files.assert_called_with(
            {
                'output_file': ('/new/path/foo.txt', None),
                'output_file_2': ('/path/to/foo_2.txt', None),
            }, job_exe)
Пример #22
0
    def test_media_type_warning(self):
        """Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a warning for a mis-matched media type"""

        rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
        rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))

        interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
        job_interface = JobInterface(json.loads(interface_json_str))

        warnings = rule_config.validate_trigger_for_job(job_interface)

        self.assertEqual(len(warnings), 1)
Пример #23
0
    def test_successful(self):
        """Tests calling ParseTriggerRuleConfiguration.validate_trigger_for_job() successfully with no warnings"""

        rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
        rule_config = ParseTriggerRuleConfiguration(PARSE_TYPE, json.loads(rule_json_str))

        interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
        job_interface = JobInterface(json.loads(interface_json_str))

        warnings = rule_config.validate_trigger_for_job(job_interface)

        self.assertListEqual(warnings, [])
Пример #24
0
    def test_output_dir_in_command(self):
        job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
        job_interface_dict['command_arguments'] = '${job_output_dir}'

        job_interface = JobInterface(job_interface_dict)
        job_data = JobData(job_data_dict)
        job_environment = job_environment_dict
        job_exe_id = 1
        job_output_dir = SCALE_JOB_EXE_OUTPUT_PATH

        job_interface.perform_pre_steps(job_data, job_environment)
        job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
        self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
Пример #25
0
 def test_command_param_will_pass_with_input(self):
     definition = {
         'command': 'test-command',
         'command_arguments': '${param-1}',
         'version': '1.0',
         'input_data': [{
             'name': 'param-1',
             'type': 'file',
         }]
     }
     try:
         JobInterface(definition)
     except InvalidInterfaceDefinition:
         self.fail('Valid definition raised a validation exception')
Пример #26
0
 def test_command_string_allows_special_formats(self):
     definition = {
         'command': 'test-command',
         'command_arguments': '${-f :param-1}',
         'version': '1.0',
         'input_data': [{
             'name': 'param-1',
             'type': 'file',
         }]
     }
     try:
         JobInterface(definition)
     except InvalidInterfaceDefinition:
         self.fail('Valid definition raised a validation exception')
Пример #27
0
 def test_input_data_must_have_a_type(self):
     definition = {
         'command': 'test-command',
         'command_arguments': '${param-1}',
         'version': '1.0',
         'input_data': [{
             'name': 'param-1',
         }]
     }
     try:
         JobInterface(definition)
         self.fail('Expected invalid job definition to throw an exception')
     except InvalidInterfaceDefinition:
         pass
Пример #28
0
 def test_interface_with_share_resource_works(self):
     definition = {
         'command': 'test-command',
         'command_arguments': '',
         'version': '1.0',
         'shared_resources': [{
             'name': 'resource-1',
             'type': 'db-connection',
         }]
     }
     try:
         JobInterface(definition)
     except InvalidInterfaceDefinition:
         self.fail('Valid definition raised a validation exception')
Пример #29
0
    def create(interface_dict, do_validate=True):
        """Instantiate an instance of the JobInterface based on inferred type

        :param interface_dict: deserialized JSON interface
        :type interface_dict: dict
        :param do_validate: whether schema validation should be applied
        :type do_validate: bool
        :return: instance of the job interface appropriate for input data
        :rtype: :class:`job.configuration.interface.job_interface.JobInterface` or
                :class:`job.seed.manifest.SeedManifest`
        """
        if JobInterfaceSunset.is_seed_dict(interface_dict):
            return SeedManifest(interface_dict, do_validate=do_validate)
        else:
            return JobInterface(interface_dict, do_validate=do_validate)
Пример #30
0
 def test_command_string_special_formats_should_have_dollar_sign(self):
     definition = {
         'command': 'test-command',
         'command_arguments': '${param-1:-f param-1}',
         'version': '1.0',
         'input_data': [{
             'name': 'param-1',
             'type': 'file',
         }]
     }
     try:
         JobInterface(definition)
         self.fail('Expected invalid job definition to throw an exception')
     except InvalidInterfaceDefinition:
         pass