def test_input_data_required_must_be_true_or_false(self): definition = { 'command': 'test-command', 'command_arguments': '${param-1}', 'version': '1.0', 'input_data': [{ 'name': 'param-1', 'type': 'file', 'required': True, }] } try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('Valid definition raised a validation exception') definition['input_data'][0]['required'] = False try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('Valid definition raised a validation exception') definition['input_data'][0]['required'] = 'some_string' try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass
def test_no_workspace_needed(self): """Tests calling JobInterface.validate_connection() without a workspace, but none is needed.""" job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.1', 'input_data': [{ 'name': 'Input 1', 'type': 'property', }, { 'name': 'Input 2', 'type': 'file', 'media_types': ['text/plain'], 'partial': True }], 'output_data': [], } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property('Input 1') job_conn.add_input_file('Input 2', False, ['text/plain'], False, True) # No exception is success job_interface.validate_connection(job_conn)
def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_isdir): def new_retrieve(arg1): return { 'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'], } mock_retrieve_call.side_effect = new_retrieve job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${files1}' job_interface_dict['input_data'] = [{ 'name': 'files1', 'type': 'files', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'files1', 'file_ids': [1, 2, 3], }) job_data_dict['output_data'].append({ 'name': 'files1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1') self.assertEqual(job_command_arguments, expected_command_arguments, 'expected a different command from pre_steps')
def test_output_file(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_file', 'workspace_id': self.workspace.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/some/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_file': ('/some/path/foo.txt', None), }, job_exe)
def test_successful(self): """Tests calling JobInterface.validate_connection() successfully.""" job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.0', 'input_data': [{ 'name': 'Input 1', 'type': 'property', }, { 'name': 'Input 2', 'type': 'file', 'media_types': ['text/plain'] }], 'output_data': [{ 'name': 'Output 1', 'type': 'file', }] } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property('Input 1') job_conn.add_input_file('Input 2', False, ['text/plain'], False, False) job_conn.add_workspace() # No exception is success job_interface.validate_connection(job_conn)
def test_invalid_output_files(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_files', 'paths': ['/some/path/foo.txt', '/other/path/foo.txt'], }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = False job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' self.assertRaises(InvalidResultsManifest, job_interface.perform_post_steps, job_exe, job_data, fake_stdout)
def test_required_workspace_missing(self): """Tests calling JobInterface.validate_connection() when a required workspace is missing""" job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.1', 'input_data': [{ 'name': 'Input 1', 'type': 'property', }, { 'name': 'Input 2', 'type': 'file', 'media_types': ['text/plain'], 'partial': True }], 'output_data': [{ 'name': 'Output 1', 'type': 'file', }] } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property('Input 1') job_conn.add_input_file('Input 2', False, ['text/plain'], False, True) self.assertRaises(InvalidConnection, job_interface.validate_connection, job_conn)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_isdir): job_exe_id = 1 def new_retrieve(arg1): return { 'file1_out': [input_file_path], } input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda (arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${file1}' job_interface_dict['input_data'] = [{ 'name': 'file1', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'file1', 'file_id': self.file.id, }) job_data_dict['output_data'].append({ 'name': 'file1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def get_job_interface(self): """Returns the interface for this queued job :returns: The job interface :rtype: :class:`job.configuration.interface.job_interface.JobInterface` """ return JobInterface(self.interface, do_validate=False)
def job_type_get_job_interface(self): """Returns the interface for running jobs of this type :returns: The job interface for this type :rtype: :class:`job.configuration.interface.job_interface.JobInterface` """ return JobInterface(self.interface)
def job_get_job_interface(self): """Returns the interface for this job :returns: The interface for this job :rtype: :class:`job.configuration.interface.job_interface.JobInterface` """ return JobInterface(self.job_type_rev.interface)
def get_job_types(recipe_types=None, job_type_ids=None, job_type_names=None, job_type_categories=None): """Exports all the job types in the system based on the given filters. :param recipe_types: Only include job types that are referenced by the given recipe types. :type recipe_types: list[:class:`recipe.models.RecipeType`] :param job_type_ids: A list of unique job type identifiers to include. :type job_type_ids: list[str] :param job_type_names: A list of job type system names to include. :type job_type_names: list[str] :param job_type_categories: A list of job type category names to include. :type job_type_categories: list[str] :returns: A list of matching job types. :rtype: list[:class:`job.models.JobType`] """ # Build a set of job type keys referenced by the recipe types job_type_keys = set() if recipe_types and not (job_type_ids or job_type_names or job_type_categories): for recipe_type in recipe_types: job_type_keys.update( recipe_type.get_recipe_definition().get_job_type_keys()) if not job_type_keys: return [] # System job types should never be exported job_types = JobType.objects.exclude( category='system').select_related('trigger_rule') # Filter by the referenced job type keys job_type_filters = [] for job_type_key in job_type_keys: job_type_filter = Q(name=job_type_key[0], version=job_type_key[1]) job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter if job_type_filters: job_types = job_types.filter(job_type_filters) # Filter by additional passed arguments if job_type_ids: job_types = job_types.filter(id__in=job_type_ids) if job_type_names: job_types = job_types.filter(name__in=job_type_names) if job_type_categories: job_types = job_types.filter(category__in=job_type_categories) # Scrub configuration for secrets for job_type in job_types: if job_type.configuration: configuration = JobConfigurationV2(job_type.configuration) interface = JobInterface(job_type.manifest) configuration.validate(interface.get_dict()) job_type.configuration = configuration.get_dict() return job_types
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) geo_metadata = { 'data_started': '2015-05-15T10:34:12Z', 'data_ended': '2015-05-15T10:36:12Z', 'geo_json': { 'type': 'Polygon', 'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]], } } results_manifest = { 'version': '1.1', 'output_data': [{ 'name': 'output_files', 'files': [{ 'path': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }, { 'path': '/other/path/foo.txt', 'geo_metadata': geo_metadata, }] }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( { 'output_files': [ ('/some/path/foo.txt', None, geo_metadata), ('/other/path/foo.txt', None, geo_metadata), ] }, job_exe)
def test_interface_must_have_command(self): definition = { 'version': '1.0', } try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass
def test_minimal_input_validation(self): definition = { 'command': 'test-command', 'command_arguments': 'some_argument', 'version': '1.0', } try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('A valid definition should not raise an Exception')
def test_bad_input_name(self): """Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a bad input name""" rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}' rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str)) interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "different_input_name", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}' job_interface = JobInterface(json.loads(interface_json_str)) self.assertRaises(InvalidConnection, rule_config.validate_trigger_for_job, job_interface)
def test_output_name_appropriate(self): good_names = [ 'foo', 'bar', 'baz', 'a file with spaces', 'file_with_underscores' ] bad_names = [ 'ca$h_money', 'do|do_not', 'try!=found', 'this_file_is_over_255_characters_long_12345678901234567890123456789012345678901234567890123456789012345678' '9012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234' '56789012345678901234567890123456789012345678901234567890!' ] definition = { 'command': 'test-command', 'command_arguments': '', 'version': '1.0', 'input_data': [{ 'name': 'foo', 'type': 'file', 'required': True, }], 'output_data': [{ 'name': 'some_output', 'type': 'file', }] } for output_name in good_names: definition['output_data'][0]['name'] = output_name try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail( 'Unable to parse a good interface definition with output name: %s' % output_name) for output_name in bad_names: definition['output_data'][0]['name'] = output_name try: JobInterface(definition) self.fail( 'job interface with a bad output name (%s) was able to get past validation' % output_name) except InvalidInterfaceDefinition: pass
def test_simple_case(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_populate_mounts(self): """Tests the addition of mount volumes to the configuration.""" exe_config = ExecutionConfiguration() config_dict = { 'version': '2.0', 'mounts': { 'mount_1': { 'type': 'host', 'host_path': '/host/path' }, 'mount_2': { 'type': 'volume', 'driver': 'x-driver', 'driver_opts': { 'foo': 'bar' } } } } interface_dict = { 'version': '1.4', 'command': 'the cmd', 'command_arguments': 'foo', 'mounts': [{ 'name': 'mount_1', 'path': '/mount_1', 'mode': 'ro' }, { 'name': 'mount_2', 'path': '/mount_2', 'mode': 'rw' }] } job_exe = MagicMock() job_exe.get_job_configuration.return_value = JobConfiguration( config_dict) job_exe.get_job_interface.return_value = JobInterface(interface_dict) job_exe.get_cluster_id.return_value = 'scale_1234' exe_config.populate_mounts(job_exe) docker_params = exe_config.get_job_task_docker_params() self.assertEqual(docker_params[0].flag, 'volume') self.assertEqual(docker_params[0].value, '/host/path:/mount_1:ro') self.assertEqual(docker_params[1].flag, 'volume') mount_2 = '$(docker volume create --name scale_1234_mount_mount_2 --driver x-driver --opt foo=bar):/mount_2:rw' self.assertEqual(docker_params[1].value, mount_2)
def test_command_param_will_fail_without_input(self): definition = { 'command': 'test-command', 'command_arguments': '${param-1}', 'version': '1.0', } try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass
def test_manifest_overrides_stdout(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }, { 'name': 'output_file_2', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/new/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = """ This text is supposed to mimic the output of a program we should see artifacts registered with the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line so the example above won't match, but this will ARTIFACT:output_file:/path/to/foo.txt We should also be able to have text after the artifact and multiple artifacts. ARTIFACT:output_file_2:/path/to/foo_2.txt """ job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( { 'output_file': ('/new/path/foo.txt', None), 'output_file_2': ('/path/to/foo_2.txt', None), }, job_exe)
def test_media_type_warning(self): """Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a warning for a mis-matched media type""" rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}' rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str)) interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}' job_interface = JobInterface(json.loads(interface_json_str)) warnings = rule_config.validate_trigger_for_job(job_interface) self.assertEqual(len(warnings), 1)
def test_successful(self): """Tests calling ParseTriggerRuleConfiguration.validate_trigger_for_job() successfully with no warnings""" rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}' rule_config = ParseTriggerRuleConfiguration(PARSE_TYPE, json.loads(rule_json_str)) interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}' job_interface = JobInterface(json.loads(interface_json_str)) warnings = rule_config.validate_trigger_for_job(job_interface) self.assertListEqual(warnings, [])
def test_output_dir_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${job_output_dir}' job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_output_dir = SCALE_JOB_EXE_OUTPUT_PATH job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_command_param_will_pass_with_input(self): definition = { 'command': 'test-command', 'command_arguments': '${param-1}', 'version': '1.0', 'input_data': [{ 'name': 'param-1', 'type': 'file', }] } try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('Valid definition raised a validation exception')
def test_command_string_allows_special_formats(self): definition = { 'command': 'test-command', 'command_arguments': '${-f :param-1}', 'version': '1.0', 'input_data': [{ 'name': 'param-1', 'type': 'file', }] } try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('Valid definition raised a validation exception')
def test_input_data_must_have_a_type(self): definition = { 'command': 'test-command', 'command_arguments': '${param-1}', 'version': '1.0', 'input_data': [{ 'name': 'param-1', }] } try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass
def test_interface_with_share_resource_works(self): definition = { 'command': 'test-command', 'command_arguments': '', 'version': '1.0', 'shared_resources': [{ 'name': 'resource-1', 'type': 'db-connection', }] } try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('Valid definition raised a validation exception')
def create(interface_dict, do_validate=True): """Instantiate an instance of the JobInterface based on inferred type :param interface_dict: deserialized JSON interface :type interface_dict: dict :param do_validate: whether schema validation should be applied :type do_validate: bool :return: instance of the job interface appropriate for input data :rtype: :class:`job.configuration.interface.job_interface.JobInterface` or :class:`job.seed.manifest.SeedManifest` """ if JobInterfaceSunset.is_seed_dict(interface_dict): return SeedManifest(interface_dict, do_validate=do_validate) else: return JobInterface(interface_dict, do_validate=do_validate)
def test_command_string_special_formats_should_have_dollar_sign(self): definition = { 'command': 'test-command', 'command_arguments': '${param-1:-f param-1}', 'version': '1.0', 'input_data': [{ 'name': 'param-1', 'type': 'file', }] } try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass