def test_input_data_required_must_be_true_or_false(self): definition = { 'command': 'test-command', 'command_arguments': '${param-1}', 'version': '1.0', 'input_data': [{ 'name': 'param-1', 'type': 'file', 'required': True, }] } try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('Valid definition raised a validation exception') definition['input_data'][0]['required'] = False try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('Valid definition raised a validation exception') definition['input_data'][0]['required'] = 'some_string' try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass
def test_no_workspace_needed(self): """Tests calling JobInterface.validate_connection() without a workspace, but none is needed.""" job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.1', 'input_data': [{ 'name': 'Input 1', 'type': 'property', }, { 'name': 'Input 2', 'type': 'file', 'media_types': ['text/plain'], 'partial': True }], 'output_data': [], } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property('Input 1') job_conn.add_input_file('Input 2', False, ['text/plain'], False, True) # No exception is success job_interface.validate_connection(job_conn)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file): job_exe_id = 1 def new_retrieve(arg1): return { 'file1_out': [input_file_path], } input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda(arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${file1}' job_interface_dict['input_data'] = [{ 'name': 'file1', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'file1', 'file_id': self.file.id, }) job_data_dict['output_data'].append({ 'name': 'file1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_isdir): def new_retrieve(arg1): return { 'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'], } mock_retrieve_call.side_effect = new_retrieve job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${files1}' job_interface_dict['input_data'] = [{ 'name': 'files1', 'type': 'files', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'files1', 'file_ids': [1, 2, 3], }) job_data_dict['output_data'].append({ 'name': 'files1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1') self.assertEqual(job_command_arguments, expected_command_arguments, 'expected a different command from pre_steps')
def test_successful(self): """Tests calling JobInterface.validate_connection() successfully.""" job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.0', 'input_data': [{ 'name': 'Input 1', 'type': 'property', }, { 'name': 'Input 2', 'type': 'file', 'media_types': ['text/plain'] }], 'output_data': [{ 'name': 'Output 1', 'type': 'file', }] } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property('Input 1') job_conn.add_input_file('Input 2', False, ['text/plain'], False, False) job_conn.add_workspace() # No exception is success job_interface.validate_connection(job_conn)
def test_output_files(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name':'output_files', 'paths': ['/some/path/foo.txt', '/other/path/foo.txt'], }] } mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_files': [ ('/some/path/foo.txt', None), ('/other/path/foo.txt', None), ] }, job_exe)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_setup_upload): job_work_dir = "/test" job_exe_id = 1 job_input_dir = get_job_exe_input_data_dir(job_exe_id) job_output_dir = os.path.join(job_work_dir, u'outputs') def new_retrieve(arg1, arg2, arg3): return {u'file1_out': [input_file_path]} input_file_path = os.path.join(job_input_dir, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda(arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict[u'command_arguments'] = u'${file1}' job_interface_dict[u'input_data'] = [{u'name' : u'file1', u'type' : u'file', 'required' : True}] job_data_dict[u'input_data'].append({u'name': u'file1', u'file_id': 1}) job_data_dict[u'output_data'].append({u'name': u'file1_out', u'workspace_id': self.workspace.id}) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_interface.perform_pre_steps(job_data, job_environment, job_exe_id) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, u'expected a different command from pre_steps') mock_setup_upload.assert_called_once_with(get_job_exe_output_data_dir(job_exe_id), get_job_exe_output_work_dir(job_exe_id), self.workspace)
def test_extra_products_are_fine(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}] job_interface_dict[u'output_data'] = [{"name":"output_file", "type":"file", "required": True}] job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1}) results_manifest={"version" : "1.0", "files" : [{"name":"output_file", "path":"/new/path/foo.txt"}]} mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = ''' This text is supposed to mimick the output of a program we should see artifacts registered with the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line so the example above won't match, but this will ARTIFACT:output_file:/path/to/foo.txt We should also be able to have text after the artifact and multiple artifacts. ARTIFACT:output_file_2:/path/to/foo_2.txt ''' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({u'output_file': (u'/new/path/foo.txt', None)}, job_exe)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_isdir): job_exe_id = 1 def new_retrieve(arg1): return { 'file1_out': [input_file_path], } input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda (arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${file1}' job_interface_dict['input_data'] = [{ 'name': 'file1', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'file1', 'file_id': self.file.id, }) job_data_dict['output_data'].append({ 'name': 'file1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def test_no_workspace_needed(self): '''Tests calling JobInterface.validate_connection() without a workspace, but none is needed.''' job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.0', 'input_data': [{ 'name': 'Input 1', 'type': 'property', }, { 'name': 'Input 2', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [], } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property('Input 1') job_conn.add_input_file('Input 2', False, ['text/plain'], False) # No exception is success job_interface.validate_connection(job_conn)
def test_output_file(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_file', 'workspace_id': self.workspace.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/some/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_file': ('/some/path/foo.txt', None), }, job_exe)
def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir): def new_retrieve(arg1): return { 'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'], } mock_retrieve_call.side_effect = new_retrieve job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${files1}' job_interface_dict['input_data'] = [{ 'name': 'files1', 'type': 'files', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'files1', 'file_ids': [1, 2, 3], }) job_data_dict['output_data'].append({ 'name': 'files1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1') self.assertEqual(job_command_arguments, expected_command_arguments, 'expected a different command from pre_steps')
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) geo_metadata = { 'data_started': '2015-05-15T10:34:12Z', 'data_ended': '2015-05-15T10:36:12Z', 'geo_json': { 'type': 'Polygon', 'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]], } } results_manifest = { 'version': '1.1', 'output_data': [{ 'name': 'output_files', 'files': [{ 'path': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }, { 'path': '/other/path/foo.txt', 'geo_metadata': geo_metadata, }] }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( { 'output_files': [ ('/some/path/foo.txt', None, geo_metadata), ('/other/path/foo.txt', None, geo_metadata), ] }, job_exe)
def get_job_types(recipe_types=None, job_type_ids=None, job_type_names=None, job_type_categories=None): """Exports all the job types in the system based on the given filters. :param recipe_types: Only include job types that are referenced by the given recipe types. :type recipe_types: list[:class:`recipe.models.RecipeType`] :param job_type_ids: A list of unique job type identifiers to include. :type job_type_ids: list[str] :param job_type_names: A list of job type system names to include. :type job_type_names: list[str] :param job_type_categories: A list of job type category names to include. :type job_type_categories: list[str] :returns: A list of matching job types. :rtype: list[:class:`job.models.JobType`] """ # Build a set of job type keys referenced by the recipe types job_type_keys = set() if recipe_types and not (job_type_ids or job_type_names or job_type_categories): for recipe_type in recipe_types: job_type_keys.update( recipe_type.get_recipe_definition().get_job_type_keys()) if not job_type_keys: return [] # System job types should never be exported job_types = JobType.objects.exclude( category='system').select_related('trigger_rule') # Filter by the referenced job type keys job_type_filters = [] for job_type_key in job_type_keys: job_type_filter = Q(name=job_type_key[0], version=job_type_key[1]) job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter if job_type_filters: job_types = job_types.filter(job_type_filters) # Filter by additional passed arguments if job_type_ids: job_types = job_types.filter(id__in=job_type_ids) if job_type_names: job_types = job_types.filter(name__in=job_type_names) if job_type_categories: job_types = job_types.filter(category__in=job_type_categories) # Scrub configuration for secrets for job_type in job_types: if job_type.configuration: configuration = JobConfigurationV2(job_type.configuration) interface = JobInterface(job_type.manifest) configuration.validate(interface.get_dict()) job_type.configuration = configuration.get_dict() return job_types
def test_simple_case(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_simple_case(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_manifest_overrides_stdout(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }, { 'name': 'output_file_2', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/new/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = """ This text is supposed to mimic the output of a program we should see artifacts registered with the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line so the example above won't match, but this will ARTIFACT:output_file:/path/to/foo.txt We should also be able to have text after the artifact and multiple artifacts. ARTIFACT:output_file_2:/path/to/foo_2.txt """ job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( { 'output_file': ('/new/path/foo.txt', None), 'output_file_2': ('/path/to/foo_2.txt', None), }, job_exe)
def test_output_dir_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${job_output_dir}' job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_output_dir = SCALE_JOB_EXE_OUTPUT_PATH job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_output_dir_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${job_output_dir}' job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_output_dir = file_system.get_job_exe_output_data_dir(job_exe_id) job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_manifest_overrides_stdout(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }, { 'name': 'output_file_2', 'type':'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/new/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = """ This text is supposed to mimic the output of a program we should see artifacts registered with the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line so the example above won't match, but this will ARTIFACT:output_file:/path/to/foo.txt We should also be able to have text after the artifact and multiple artifacts. ARTIFACT:output_file_2:/path/to/foo_2.txt """ job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_file': ('/new/path/foo.txt', None), 'output_file_2': ('/path/to/foo_2.txt', None), }, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) geo_metadata = { 'data_started': '2015-05-15T10:34:12Z', 'data_ended': '2015-05-15T10:36:12Z', 'geo_json': { 'type': 'Polygon', 'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]], } } results_manifest = { 'version': '1.1', 'output_data': [{ 'name': 'output_files', 'files': [{ 'path': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }, { 'path': '/other/path/foo.txt', 'geo_metadata': geo_metadata, }] }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_files': [ ('/some/path/foo.txt', None, geo_metadata), ('/other/path/foo.txt', None, geo_metadata), ] }, job_exe)
def test_successful(self): '''Tests calling JobInterface.add_output_to_connection() successfully.''' job_interface_dict = { u'command': u'simple-command', u'command_arguments': u'', u'version': u'1.0', u'input_data': [], u'output_data': [{u'name': u'Output 1', u'type': u'file'}] } job_interface = JobInterface(job_interface_dict) job_conn = MagicMock() job_interface.add_output_to_connection(u'Output 1', job_conn, u'Input 1') job_conn.add_input_file.assert_called_with(u'Input 1', False, [], False)
def test_invalid_output_files(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_files', 'paths': ['/some/path/foo.txt', '/other/path/foo.txt'], }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = False job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' self.assertRaises(InvalidResultsManifest, job_interface.perform_post_steps, job_exe, job_data, fake_stdout)
def test_required_workspace_missing(self): """Tests calling JobInterface.validate_connection() when a required workspace is missing""" job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.1', 'input_data': [{ 'name': 'Input 1', 'type': 'property', }, { 'name': 'Input 2', 'type': 'file', 'media_types': ['text/plain'], 'partial': True }], 'output_data': [{ 'name': 'Output 1', 'type': 'file', }] } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property('Input 1') job_conn.add_input_file('Input 2', False, ['text/plain'], False, True) self.assertRaises(InvalidConnection, job_interface.validate_connection, job_conn)
def test_successful(self, mock_get_dict): """Tests calling JobInterface.update() successfully.""" mock_get_dict.return_value = self.job_interface_dict job_interface = JobInterface.convert_interface(self.job_interface_dict) self.assertEqual(job_interface['version'], '1.1') self.assertIn('partial', job_interface['input_data'][0]) self.assertFalse(job_interface['input_data'][0]['partial'])
def test_complex_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict[u'command_arguments'] = u'${-f :prop1}' job_interface_dict[u'input_data'] = [{u'name' : u'prop1', u'type' : u'property', 'required' : True}] job_data_dict[u'input_data'].append({u'name': u'prop1', u'value': u'property-value'}) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_work_dir = "/test" job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, u'-f property-value', u'expected a different command from pre_steps')
def job_type_get_job_interface(self): """Returns the interface for running jobs of this type :returns: The job interface for this type :rtype: :class:`job.configuration.interface.job_interface.JobInterface` """ return JobInterface(self.interface)
def get_job_interface(self): """Returns the interface for this queued job :returns: The job interface :rtype: :class:`job.configuration.interface.job_interface.JobInterface` """ return JobInterface(self.interface, do_validate=False)
def job_get_job_interface(self): """Returns the interface for this job :returns: The interface for this job :rtype: :class:`job.configuration.interface.job_interface.JobInterface` """ return JobInterface(self.job_type_rev.interface)
def test_output_files(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}] job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1}) results_manifest={"version" : "1.0", "files" : [{"name":"output_files", "paths": ["/some/path/foo.txt", "/other/path/foo.txt"]}]} mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = u'' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({u'output_files': [(u'/some/path/foo.txt', None),(u'/other/path/foo.txt', None)]}, job_exe)
def test_successful(self): """Tests calling JobInterface.add_output_to_connection() successfully.""" job_interface_dict = { 'command': 'simple-command', 'command_arguments': '', 'version': '1.0', 'input_data': [], 'output_data': [{ 'name': 'Output 1', 'type': 'file', }] } job_interface = JobInterface(job_interface_dict) job_conn = MagicMock() job_interface.add_output_to_connection('Output 1', job_conn, 'Input 1') job_conn.add_input_file.assert_called_with('Input 1', False, [], False, False)
def test_interface_must_have_command(self): definition = { 'version': '1.0', } try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass
def test_absent_optional_files_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${input_files}' job_interface_dict['input_data'] = [{ 'name': 'input_files', 'type': 'files', 'required': False, }] job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}] job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1}) geo_metadata = { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ]]] } } results_manifest = { "version": "1.1", "output_data": [ { "name" : "output_files", "files": [ { "path" : "/some/path/foo.txt", "geo_metadata": geo_metadata }, { "path" : "/other/path/foo.txt", "geo_metadata": geo_metadata } ] } ] } mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = u'' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( {u'output_files': [(u'/some/path/foo.txt', None, geo_metadata),(u'/other/path/foo.txt', None, geo_metadata)]}, job_exe)
def test_absent_required_files_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${input_files}' job_interface_dict['input_data'] = [{ 'name': 'input_files', 'type': 'files', 'required': True, }] job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) self.assertRaises(InvalidData, lambda: job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id))
def test_bad_input_name(self): """Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a bad input name""" rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}' rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str)) interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "different_input_name", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}' job_interface = JobInterface(json.loads(interface_json_str)) self.assertRaises(InvalidConnection, rule_config.validate_trigger_for_job, job_interface)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_setup_upload): job_exe_id = 1 job_input_dir = file_system.get_job_exe_input_data_dir(job_exe_id) def new_retrieve(arg1, arg2, arg3): return { 'file1_out': [input_file_path], } input_file_path = os.path.join(job_input_dir, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda (arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env( ) job_interface_dict['command_arguments'] = '${file1}' job_interface_dict['input_data'] = [{ 'name': 'file1', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'file1', 'file_id': self.file.id, }) job_data_dict['output_data'].append({ 'name': 'file1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_interface.perform_pre_steps(job_data, job_environment, job_exe_id) job_command_arguments = job_interface.fully_populate_command_argument( job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps') mock_setup_upload.assert_called_once_with( file_system.get_job_exe_output_data_dir(job_exe_id), file_system.get_job_exe_output_work_dir(job_exe_id), self.workspace)
def test_minimal_input_validation(self): definition = { 'command': 'test-command', 'command_arguments': 'some_argument', 'version': '1.0', } try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail('A valid definition should not raise an Exception')
def test_output_name_appropriate(self): good_names = [ 'foo', 'bar', 'baz', 'a file with spaces', 'file_with_underscores' ] bad_names = [ 'ca$h_money', 'do|do_not', 'try!=found', 'this_file_is_over_255_characters_long_12345678901234567890123456789012345678901234567890123456789012345678' '9012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234' '56789012345678901234567890123456789012345678901234567890!' ] definition = { 'command': 'test-command', 'command_arguments': '', 'version': '1.0', 'input_data': [{ 'name': 'foo', 'type': 'file', 'required': True, }], 'output_data': [{ 'name': 'some_output', 'type': 'file', }] } for output_name in good_names: definition['output_data'][0]['name'] = output_name try: JobInterface(definition) except InvalidInterfaceDefinition: self.fail( 'Unable to parse a good interface definition with output name: %s' % output_name) for output_name in bad_names: definition['output_data'][0]['name'] = output_name try: JobInterface(definition) self.fail( 'job interface with a bad output name (%s) was able to get past validation' % output_name) except InvalidInterfaceDefinition: pass
def test_command_param_will_fail_without_input(self): definition = { 'command': 'test-command', 'command_arguments': '${param-1}', 'version': '1.0', } try: JobInterface(definition) self.fail('Expected invalid job definition to throw an exception') except InvalidInterfaceDefinition: pass
def test_populate_mounts(self): """Tests the addition of mount volumes to the configuration.""" exe_config = ExecutionConfiguration() config_dict = { 'version': '2.0', 'mounts': { 'mount_1': { 'type': 'host', 'host_path': '/host/path' }, 'mount_2': { 'type': 'volume', 'driver': 'x-driver', 'driver_opts': { 'foo': 'bar' } } } } interface_dict = { 'version': '1.4', 'command': 'the cmd', 'command_arguments': 'foo', 'mounts': [{ 'name': 'mount_1', 'path': '/mount_1', 'mode': 'ro' }, { 'name': 'mount_2', 'path': '/mount_2', 'mode': 'rw' }] } job_exe = MagicMock() job_exe.get_job_configuration.return_value = JobConfiguration( config_dict) job_exe.get_job_interface.return_value = JobInterface(interface_dict) job_exe.get_cluster_id.return_value = 'scale_1234' exe_config.populate_mounts(job_exe) docker_params = exe_config.get_job_task_docker_params() self.assertEqual(docker_params[0].flag, 'volume') self.assertEqual(docker_params[0].value, '/host/path:/mount_1:ro') self.assertEqual(docker_params[1].flag, 'volume') mount_2 = '$(docker volume create --name scale_1234_mount_mount_2 --driver x-driver --opt foo=bar):/mount_2:rw' self.assertEqual(docker_params[1].value, mount_2)
def test_successful(self): '''Tests calling JobInterface.validate_connection() successfully.''' job_interface_dict = { u'command': u'simple-command', u'command_arguments': u'', u'version': u'1.0', u'input_data': [{u'name': u'Input 1', u'type': u'property'}, {u'name': u'Input 2', u'type': u'file', u'media_types': [u'text/plain']}], u'output_data': [{u'name': u'Output 1', u'type': u'file'}] } job_interface = JobInterface(job_interface_dict) job_conn = JobConnection() job_conn.add_property(u'Input 1') job_conn.add_input_file(u'Input 2', False, [u'text/plain'], False) job_conn.add_workspace() # No exception is success job_interface.validate_connection(job_conn)
def test_complex_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${-f :prop1}' job_interface_dict['input_data'] = [{ 'name': 'prop1', 'type': 'property', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'prop1', 'value': 'property-value', }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '-f property-value', 'expected a different command from pre_steps')
def test_property_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${prop1}' job_interface_dict['input_data'] = [{ 'name': 'prop1', 'type': 'property', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'prop1', 'value': 'property-value', }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, 'property-value', 'expected a different command from pre_steps')
def test_successful(self): """Tests calling ParseTriggerRuleConfiguration.validate_trigger_for_job() successfully with no warnings""" rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}' rule_config = ParseTriggerRuleConfiguration(PARSE_TYPE, json.loads(rule_json_str)) interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}' job_interface = JobInterface(json.loads(interface_json_str)) warnings = rule_config.validate_trigger_for_job(job_interface) self.assertListEqual(warnings, [])
def test_media_type_warning(self): """Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a warning for a mis-matched media type""" rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}' rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str)) interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}' job_interface = JobInterface(json.loads(interface_json_str)) warnings = rule_config.validate_trigger_for_job(job_interface) self.assertEqual(len(warnings), 1)
def test_parse_data(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}] job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1}) geo_json = {"type": 'Feature'} geo_metadata = { "data_started": '2015-01-01T00:00:00Z', "geo_json": geo_json } results_manifest={"version" : "1.1", "parse_results" : [{"filename":"/some/path/foo.txt", "geo_metadata":geo_metadata}]} mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = u'' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.save_parse_results.assert_called_with({u'/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None)})
def test_parse_data(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) geo_json = {'type': 'Feature'} geo_metadata = { 'data_started': '2015-01-01T00:00:00Z', 'geo_json': geo_json } results_manifest = { 'version': '1.1', 'parse_results': [{ 'filename': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }] } mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.save_parse_results.assert_called_with({ '/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None), })
def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.error = error_test_utils.create_error() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) self.configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration) self.error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-15': self.error.name, } })
class TestJobTypeManagerEditJobType(TransactionTestCase): def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) self.job_type = JobType.objects.create_job_type('name', '1.0', self.job_interface) new_valid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['application/json'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.new_valid_job_interface = JobInterface(new_valid_interface) new_invalid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }]} self.new_invalid_job_interface = JobInterface(new_invalid_interface) self.definition = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type.name, 'version': self.job_type.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }] }] } self.recipe_def = RecipeDefinition(self.definition) self.recipe = RecipeType.objects.create_recipe_type('name', '1.0', 'title', 'description', self.recipe_def, None) def test_valid_interface(self): """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change is made""" # Call test JobType.objects.edit_job_type(self.job_type.id, self.new_valid_job_interface) # Check results job_type = JobType.objects.get(pk=self.job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_valid_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) # New revision due to interface change num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2) def test_invalid_interface(self): """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface change is made""" # Call test self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type, self.job_type.id, self.new_invalid_job_interface) # Check results job_type = JobType.objects.get(pk=self.job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1)
def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) self.job_type = JobType.objects.create_job_type('name', '1.0', self.job_interface) new_valid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['application/json'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.new_valid_job_interface = JobInterface(new_valid_interface) new_invalid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }]} self.new_invalid_job_interface = JobInterface(new_invalid_interface) self.definition = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type.name, 'version': self.job_type.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }] }] } self.recipe_def = RecipeDefinition(self.definition) self.recipe = RecipeType.objects.create_recipe_type('name', '1.0', 'title', 'description', self.recipe_def, None)
class TestJobTypeManagerEditJobType(TransactionTestCase): def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.error = error_test_utils.create_error() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) new_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }]} self.new_job_interface = JobInterface(new_interface) self.configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration) self.new_configuration = { 'version': '1.0', 'condition': { 'media_type': 'application/json' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.new_trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.new_configuration) def test_change_general_fields(self): '''Tests calling JobTypeManager.edit_job_type() with a change to some general fields''' name = 'my-job-type' version = '1.0' title = 'my title' priority = 12 error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-15': self.error.name, } }) new_title = 'my new title' new_priority = 13 new_error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-16': self.error.name, } }) new_is_paused = True trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title, priority=priority, error_mapping=error_mapping) # Call test JobType.objects.edit_job_type(job_type.id, title=new_title, priority=new_priority, error_mapping=new_error_mapping, is_paused=new_is_paused) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) self.assertEqual(job_type.title, new_title) self.assertEqual(job_type.priority, new_priority) self.assertDictEqual(job_type.get_error_interface().get_dict(), new_error_mapping.get_dict()) self.assertEqual(job_type.is_paused, new_is_paused) self.assertIsNotNone(job_type.paused) def test_change_to_interface(self): '''Tests calling JobTypeManager.edit_job_type() with a change to the interface''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, self.new_job_interface, None, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) # New revision due to interface change num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2) def test_change_to_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() with a change to the trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, None, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id) self.assertTrue(new_trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1) def test_remove_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() that removes the trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, None, None, True) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1) def test_change_to_both(self): '''Tests calling JobTypeManager.edit_job_type() with a change to both the definition and the trigger rule ''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, self.new_job_interface, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id) self.assertTrue(new_trigger_rule.is_active) # New revision due to definition change num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2) def test_invalid_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() with a new invalid trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test self.assertRaises(InvalidConnection, JobType.objects.edit_job_type, job_type.id, self.new_job_interface, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1) def test_system_job_type(self): '''Tests calling JobTypeManager.edit_job_type() for a system job type''' name = 'my-job-type' version = '1.0' title = 'my title' new_title = 'my new title' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title) job_type.is_system = True job_type.save() # Call test self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) # No change self.assertEqual(job_type.title, title) def test_uneditable_field(self): '''Tests calling JobTypeManager.edit_job_type() to change an uneditable field''' name = 'my-job-type' version = '1.0' title = 'my title' new_title = 'my new title' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title) # Call test self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title, is_system=True) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) # No change self.assertEqual(job_type.title, title) def test_invalid_error_mapping(self): '''Tests calling JobTypeManager.edit_job_type() with an invalid error mapping''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_system = True error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '1': 'test-invalid-error', } }) # Call test self.assertRaises(Exception, JobType.objects.edit_job_type, name, version, self.job_interface, error_mapping=error_mapping, title=title, description=description, priority=priority, is_system=is_system)
class TestJobTypeManagerCreateJobType(TransactionTestCase): def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.error = error_test_utils.create_error() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) self.configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration) self.error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-15': self.error.name, } }) def test_successful_no_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() successfully with no trigger rule or error mapping''' name = 'my-job-type' version = '1.0' # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule_id) self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict()) def test_successful_with_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() successfully with a trigger rule and error mapping''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, self.error_mapping) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) self.assertDictEqual(job_type.get_error_interface().get_dict(), self.error_mapping.get_dict()) def test_invalid_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() with an invalid trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE, configuration=self.trigger_config.get_dict()) # Call test self.assertRaises(InvalidConnection, JobType.objects.create_job_type, name, version, self.job_interface, trigger_rule, self.error_mapping) def test_successful_other_fields(self): '''Tests calling JobTypeManager.create_job_type() successfully with additional fields''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title, description=description, priority=priority) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule_id) self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict()) self.assertEqual(job_type.description, description) self.assertEqual(job_type.priority, priority) self.assertIsNone(job_type.archived) self.assertIsNone(job_type.paused) def test_successful_paused(self): '''Tests calling JobTypeManager.create_job_type() and pausing it''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_paused = True # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title, description=description, priority=priority, is_paused=is_paused) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule_id) self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict()) self.assertEqual(job_type.description, description) self.assertEqual(job_type.priority, priority) self.assertEqual(job_type.is_paused, is_paused) self.assertIsNotNone(job_type.paused) def test_uneditable_field(self): '''Tests calling JobTypeManager.create_job_type() with an uneditable field''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_system = True # Call test self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, title=title, description=description, priority=priority, is_system=is_system) def test_invalid_error_mapping(self): '''Tests calling JobTypeManager.create_job_type() with an invalid error mapping''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_system = True error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '1': 'test-invalid-error', } }) # Call test self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, error_mapping=error_mapping, title=title, description=description, priority=priority, is_system=is_system)