def get_job_types(recipe_types=None, job_type_ids=None, job_type_names=None, job_type_categories=None): """Exports all the job types in the system based on the given filters. :param recipe_types: Only include job types that are referenced by the given recipe types. :type recipe_types: list[:class:`recipe.models.RecipeType`] :param job_type_ids: A list of unique job type identifiers to include. :type job_type_ids: list[str] :param job_type_names: A list of job type system names to include. :type job_type_names: list[str] :param job_type_categories: A list of job type category names to include. :type job_type_categories: list[str] :returns: A list of matching job types. :rtype: list[:class:`job.models.JobType`] """ # Build a set of job type keys referenced by the recipe types job_type_keys = set() if recipe_types and not (job_type_ids or job_type_names or job_type_categories): for recipe_type in recipe_types: job_type_keys.update( recipe_type.get_recipe_definition().get_job_type_keys()) if not job_type_keys: return [] # System job types should never be exported job_types = JobType.objects.exclude( category='system').select_related('trigger_rule') # Filter by the referenced job type keys job_type_filters = [] for job_type_key in job_type_keys: job_type_filter = Q(name=job_type_key[0], version=job_type_key[1]) job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter if job_type_filters: job_types = job_types.filter(job_type_filters) # Filter by additional passed arguments if job_type_ids: job_types = job_types.filter(id__in=job_type_ids) if job_type_names: job_types = job_types.filter(name__in=job_type_names) if job_type_categories: job_types = job_types.filter(category__in=job_type_categories) # Scrub configuration for secrets for job_type in job_types: if job_type.configuration: configuration = JobConfigurationV2(job_type.configuration) interface = JobInterface(job_type.manifest) configuration.validate(interface.get_dict()) job_type.configuration = configuration.get_dict() return job_types
class TestJobTypeManagerEditJobType(TransactionTestCase): def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) self.job_type = JobType.objects.create_job_type('name', '1.0', self.job_interface) new_valid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['application/json'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.new_valid_job_interface = JobInterface(new_valid_interface) new_invalid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }]} self.new_invalid_job_interface = JobInterface(new_invalid_interface) self.definition = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type.name, 'version': self.job_type.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }] }] } self.recipe_def = RecipeDefinition(self.definition) self.recipe = RecipeType.objects.create_recipe_type('name', '1.0', 'title', 'description', self.recipe_def, None) def test_valid_interface(self): """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change is made""" # Call test JobType.objects.edit_job_type(self.job_type.id, self.new_valid_job_interface) # Check results job_type = JobType.objects.get(pk=self.job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_valid_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) # New revision due to interface change num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2) def test_invalid_interface(self): """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface change is made""" # Call test self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type, self.job_type.id, self.new_invalid_job_interface) # Check results job_type = JobType.objects.get(pk=self.job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1)
class TestJobTypeManagerEditJobType(TransactionTestCase): def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.error = error_test_utils.create_error() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) new_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }]} self.new_job_interface = JobInterface(new_interface) self.configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration) self.new_configuration = { 'version': '1.0', 'condition': { 'media_type': 'application/json' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.new_trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.new_configuration) def test_change_general_fields(self): '''Tests calling JobTypeManager.edit_job_type() with a change to some general fields''' name = 'my-job-type' version = '1.0' title = 'my title' priority = 12 error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-15': self.error.name, } }) new_title = 'my new title' new_priority = 13 new_error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-16': self.error.name, } }) new_is_paused = True trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title, priority=priority, error_mapping=error_mapping) # Call test JobType.objects.edit_job_type(job_type.id, title=new_title, priority=new_priority, error_mapping=new_error_mapping, is_paused=new_is_paused) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) self.assertEqual(job_type.title, new_title) self.assertEqual(job_type.priority, new_priority) self.assertDictEqual(job_type.get_error_interface().get_dict(), new_error_mapping.get_dict()) self.assertEqual(job_type.is_paused, new_is_paused) self.assertIsNotNone(job_type.paused) def test_change_to_interface(self): '''Tests calling JobTypeManager.edit_job_type() with a change to the interface''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, self.new_job_interface, None, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) # New revision due to interface change num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2) def test_change_to_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() with a change to the trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, None, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id) self.assertTrue(new_trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1) def test_remove_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() that removes the trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, None, None, True) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1) def test_change_to_both(self): '''Tests calling JobTypeManager.edit_job_type() with a change to both the definition and the trigger rule ''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, self.new_job_interface, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id) self.assertTrue(new_trigger_rule.is_active) # New revision due to definition change num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2) def test_invalid_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() with a new invalid trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test self.assertRaises(InvalidConnection, JobType.objects.edit_job_type, job_type.id, self.new_job_interface, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1) def test_system_job_type(self): '''Tests calling JobTypeManager.edit_job_type() for a system job type''' name = 'my-job-type' version = '1.0' title = 'my title' new_title = 'my new title' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title) job_type.is_system = True job_type.save() # Call test self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) # No change self.assertEqual(job_type.title, title) def test_uneditable_field(self): '''Tests calling JobTypeManager.edit_job_type() to change an uneditable field''' name = 'my-job-type' version = '1.0' title = 'my title' new_title = 'my new title' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title) # Call test self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title, is_system=True) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) # No change self.assertEqual(job_type.title, title) def test_invalid_error_mapping(self): '''Tests calling JobTypeManager.edit_job_type() with an invalid error mapping''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_system = True error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '1': 'test-invalid-error', } }) # Call test self.assertRaises(Exception, JobType.objects.edit_job_type, name, version, self.job_interface, error_mapping=error_mapping, title=title, description=description, priority=priority, is_system=is_system)
class TestJobTypeManagerCreateJobType(TransactionTestCase): def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.error = error_test_utils.create_error() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(interface) self.configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration) self.error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-15': self.error.name, } }) def test_successful_no_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() successfully with no trigger rule or error mapping''' name = 'my-job-type' version = '1.0' # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule_id) self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict()) def test_successful_with_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() successfully with a trigger rule and error mapping''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, self.error_mapping) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) self.assertDictEqual(job_type.get_error_interface().get_dict(), self.error_mapping.get_dict()) def test_invalid_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() with an invalid trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE, configuration=self.trigger_config.get_dict()) # Call test self.assertRaises(InvalidConnection, JobType.objects.create_job_type, name, version, self.job_interface, trigger_rule, self.error_mapping) def test_successful_other_fields(self): '''Tests calling JobTypeManager.create_job_type() successfully with additional fields''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title, description=description, priority=priority) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule_id) self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict()) self.assertEqual(job_type.description, description) self.assertEqual(job_type.priority, priority) self.assertIsNone(job_type.archived) self.assertIsNone(job_type.paused) def test_successful_paused(self): '''Tests calling JobTypeManager.create_job_type() and pausing it''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_paused = True # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title, description=description, priority=priority, is_paused=is_paused) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule_id) self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict()) self.assertEqual(job_type.description, description) self.assertEqual(job_type.priority, priority) self.assertEqual(job_type.is_paused, is_paused) self.assertIsNotNone(job_type.paused) def test_uneditable_field(self): '''Tests calling JobTypeManager.create_job_type() with an uneditable field''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_system = True # Call test self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, title=title, description=description, priority=priority, is_system=is_system) def test_invalid_error_mapping(self): '''Tests calling JobTypeManager.create_job_type() with an invalid error mapping''' name = 'my-job-type' version = '1.0' title = 'my title' description = 'my-description' priority = 13 is_system = True error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '1': 'test-invalid-error', } }) # Call test self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, error_mapping=error_mapping, title=title, description=description, priority=priority, is_system=is_system)
class TestJobTypeManagerEditJobType(TransactionTestCase): def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }] } self.job_interface = JobInterface(interface) self.job_type = JobType.objects.create_job_type( 'name', '1.0', self.job_interface) new_valid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['application/json'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }] } self.new_valid_job_interface = JobInterface(new_valid_interface) new_invalid_interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }] } self.new_invalid_job_interface = JobInterface(new_invalid_interface) self.definition = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type.name, 'version': self.job_type.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }] }] } self.recipe_def = RecipeDefinition(self.definition) self.recipe = RecipeType.objects.create_recipe_type( 'name', '1.0', 'title', 'description', self.recipe_def, None) def test_valid_interface(self): """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change is made""" # Call test JobType.objects.edit_job_type(self.job_type.id, self.new_valid_job_interface) # Check results job_type = JobType.objects.get(pk=self.job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_valid_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) # New revision due to interface change num_of_revs = JobTypeRevision.objects.filter( job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2) def test_invalid_interface(self): """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface change is made""" # Call test self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type, self.job_type.id, self.new_invalid_job_interface) # Check results job_type = JobType.objects.get(pk=self.job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) num_of_revs = JobTypeRevision.objects.filter( job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1)
def _import_job_type(job_type_dict, job_type=None, validating=False): """Attempts to apply the given job types configuration to the system. Note that proper model locking must be performed before calling this method. :param job_type_dict: A dictionary of job type configuration changes to import. :type job_type_dict: dict :param job_type: The existing job type model to update if applicable. :type job_type: :class:`job.models.JobType` :param validating: Flag to determine if running a validate or commit transaction. :type validating: bool :returns: A list of warnings discovered during import. :rtype: list[:class:`port.schema.ValidationWarning`] :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification. """ warnings = [] # Parse the JSON content into validated model fields job_type_serializer = serializers.ConfigurationJobTypeSerializer(job_type, data=job_type_dict) if not job_type_serializer.is_valid(): raise InvalidConfiguration('Invalid job type schema: %s -> %s' % (job_type_dict['name'], job_type_serializer.errors)) result = job_type_serializer.validated_data # Importing system-level job types is not allowed if result.get('category') == 'system' or (job_type and job_type.category == 'system'): raise InvalidConfiguration('System job types cannot be imported: %s' % result.get('name')) # Validate the job interface try: interface_dict = None if 'interface' in result: interface_dict = result.get('interface') elif job_type: interface_dict = job_type.manifest interface = JobInterface(interface_dict) except InvalidInterfaceDefinition as ex: raise InvalidConfiguration('Job type interface invalid: %s -> %s' % (result.get('name'), unicode(ex))) # Validate the job configuration try: configuration_dict = None secrets = None if 'configuration' in result: configuration_dict = result.get('configuration') elif job_type: configuration_dict = job_type.configuration if interface: configuration = JobConfigurationV2(configuration_dict) if not validating: secrets = configuration.get_secret_settings(interface.get_dict()) warnings.extend(configuration.validate(interface.get_dict())) except InvalidJobConfiguration as ex: raise InvalidConfiguration('Job type configuration invalid: %s -> %s' % (result.get('name'), unicode(ex))) # Validate the error mapping try: error_mapping_dict = None if 'error_mapping' in result: error_mapping_dict = result.get('error_mapping') elif job_type: error_mapping_dict = job_type.error_mapping error_mapping = create_legacy_error_mapping(error_mapping_dict) error_mapping.validate_legacy() except InvalidInterfaceDefinition as ex: raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' % (result.get('name'), unicode(ex))) # Validate the trigger rule trigger_rule = None if 'trigger_rule' in result and result.get('trigger_rule'): trigger_rule = TriggerRule(**result.get('trigger_rule')) if trigger_rule: trigger_config = trigger_rule.get_configuration() if not isinstance(trigger_config, JobTriggerRuleConfiguration): logger.exception('Job type trigger rule type invalid') raise InvalidConfiguration('Job type trigger type invalid: %s -> %s' % (result.get('name'), trigger_rule.type)) try: warnings.extend(trigger_config.validate_trigger_for_job(interface)) # Create a new rule when the trigger content was provided if job_type_dict.get('trigger_rule'): rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type) trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name, trigger_rule.is_active) except (InvalidTriggerType, InvalidTriggerRule, InvalidConnection) as ex: logger.exception('Job type trigger rule invalid') raise InvalidConfiguration('Job type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex))) remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict['trigger_rule'] # Extract the fields that should be updated as keyword arguments extra_fields = {} base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping', 'configuration'} for key in job_type_dict: if key not in base_fields: if key in JobType.UNEDITABLE_FIELDS: warnings.append(ValidationWarning( 'read-only', 'Job type includes read-only field: %s -> %s' % (result.get('name'), key) )) else: extra_fields[key] = job_type_dict[key] # Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API if 'mem_required' in extra_fields: extra_fields['mem_const_required'] = extra_fields['mem_required'] del extra_fields['mem_required'] # Edit or create the associated job type model if job_type: try: JobType.objects.edit_job_type_v5(job_type.id, interface=interface, trigger_rule=trigger_rule, remove_trigger_rule=remove_trigger_rule, error_mapping=error_mapping, configuration=configuration, secrets=secrets, **extra_fields) except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition, InvalidSecretsConfiguration) as ex: logger.exception('Job type edit failed') raise InvalidConfiguration( 'Unable to edit existing job type: %s -> %s' % (result.get('name'), unicode(ex))) else: try: JobType.objects.create_job_type_v5(name=result.get('name'), version=result.get('version'), interface=interface, trigger_rule=trigger_rule, error_mapping=error_mapping, configuration=configuration, secrets=secrets, **extra_fields) except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition, InvalidSecretsConfiguration) as ex: logger.exception('Job type create failed') raise InvalidConfiguration('Unable to create new job type: %s -> %s' % (result.get('name'), unicode(ex))) return warnings
def create(self, request): """Creates a new job type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') version = rest_util.parse_string(request, 'version') # Validate the job interface interface_dict = rest_util.parse_dict(request, 'interface') interface = None try: if interface_dict: interface = JobInterface(interface_dict) except InvalidInterfaceDefinition as ex: raise BadParameter('Job type interface invalid: %s' % unicode(ex)) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None secrets = None try: if configuration_dict: configuration = JobConfiguration(configuration_dict) secrets = configuration.get_secret_settings( interface.get_dict()) configuration.validate(interface.get_dict()) except InvalidJobConfiguration as ex: raise BadParameter('Job type configuration invalid: %s' % unicode(ex)) # Validate the error mapping error_dict = rest_util.parse_dict(request, 'error_mapping', required=False) error_mapping = None try: if error_dict: error_mapping = ErrorInterface(error_dict) error_mapping.validate() except InvalidInterfaceDefinition as ex: raise BadParameter('Job type error mapping invalid: %s' % unicode(ex)) # Validate the custom resources resources_dict = rest_util.parse_dict(request, 'custom_resources', required=False) custom_resources = None try: if resources_dict: custom_resources = Resources(resources_dict) except InvalidResources as ex: raise BadParameter('Job type custom resources invalid: %s' % unicode(ex)) # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter( 'Trigger type and configuration are required together.') is_active = trigger_rule_dict[ 'is_active'] if 'is_active' in trigger_rule_dict else True # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler( trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception('Invalid trigger type for new job type: %s', name) raise BadParameter(unicode(ex)) # Extract the fields that should be updated as keyword arguments extra_fields = {} base_fields = { 'name', 'version', 'interface', 'trigger_rule', 'error_mapping', 'custom_resources', 'configuration' } for key, value in request.data.iteritems(): if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS: extra_fields[key] = value # Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API if 'mem_required' in extra_fields: extra_fields['mem_const_required'] = extra_fields['mem_required'] del extra_fields['mem_required'] try: with transaction.atomic(): # Attempt to create the trigger rule trigger_rule = None if rule_handler and 'configuration' in trigger_rule_dict: trigger_rule = rule_handler.create_trigger_rule( trigger_rule_dict['configuration'], name, is_active) # Create the job type job_type = JobType.objects.create_job_type( name=name, version=version, interface=interface, trigger_rule=trigger_rule, error_mapping=error_mapping, custom_resources=custom_resources, configuration=configuration, secrets=secrets, **extra_fields) except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidSecretsConfiguration, ValueError) as ex: logger.exception('Unable to create new job type: %s', name) raise BadParameter(unicode(ex)) # Fetch the full job type with details try: job_type = JobType.objects.get_details(job_type.id) except JobType.DoesNotExist: raise Http404 url = reverse('job_type_details_view', args=[job_type.id], request=request) serializer = JobTypeDetailsSerializer(job_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def patch(self, request, job_type_id): """Edits an existing job type and returns the updated details :param request: the HTTP PATCH request :type request: :class:`rest_framework.request.Request` :param job_type_id: The ID for the job type. :type job_type_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ # Validate the job interface interface_dict = rest_util.parse_dict(request, 'interface', required=False) interface = None try: if interface_dict: interface = JobInterface(interface_dict) except InvalidInterfaceDefinition as ex: raise BadParameter('Job type interface invalid: %s' % unicode(ex)) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None secrets = None try: if configuration_dict: configuration = JobConfiguration(configuration_dict) if interface: secrets = configuration.get_secret_settings( interface.get_dict()) configuration.validate(interface.get_dict()) else: stored_interface = JobType.objects.values_list( 'interface', flat=True).get(pk=job_type_id) secrets = configuration.get_secret_settings( stored_interface) configuration.validate(stored_interface) except InvalidJobConfiguration as ex: raise BadParameter('Job type configuration invalid: %s' % unicode(ex)) # Validate the error mapping error_dict = rest_util.parse_dict(request, 'error_mapping', required=False) error_mapping = None try: if error_dict: error_mapping = ErrorInterface(error_dict) error_mapping.validate() except InvalidInterfaceDefinition as ex: raise BadParameter('Job type error mapping invalid: %s' % unicode(ex)) # Validate the custom resources resources_dict = rest_util.parse_dict(request, 'custom_resources', required=False) custom_resources = None try: if resources_dict: custom_resources = Resources(resources_dict) except InvalidResources as ex: raise BadParameter('Job type custom resources invalid: %s' % unicode(ex)) # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter( 'Trigger type and configuration are required together.') is_active = trigger_rule_dict[ 'is_active'] if 'is_active' in trigger_rule_dict else True remove_trigger_rule = rest_util.has_params( request, 'trigger_rule') and not trigger_rule_dict # Fetch the current job type model try: job_type = JobType.objects.select_related('trigger_rule').get( pk=job_type_id) except JobType.DoesNotExist: raise Http404 # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler( trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception('Invalid trigger type for job type: %i', job_type_id) raise BadParameter(unicode(ex)) # Extract the fields that should be updated as keyword arguments extra_fields = {} base_fields = { 'name', 'version', 'interface', 'trigger_rule', 'error_mapping', 'custom_resources', 'configuration' } for key, value in request.data.iteritems(): if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS: extra_fields[key] = value # Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API if 'mem_required' in extra_fields: extra_fields['mem_const_required'] = extra_fields['mem_required'] del extra_fields['mem_required'] try: from recipe.configuration.definition.exceptions import InvalidDefinition except: logger.exception( 'Failed to import higher level recipe application.') pass try: with transaction.atomic(): # Attempt to create the trigger rule trigger_rule = None if rule_handler and 'configuration' in trigger_rule_dict: trigger_rule = rule_handler.create_trigger_rule( trigger_rule_dict['configuration'], job_type.name, is_active) # Update the active state separately if that is only given trigger field if not trigger_rule and job_type.trigger_rule and 'is_active' in trigger_rule_dict: job_type.trigger_rule.is_active = is_active job_type.trigger_rule.save() # Edit the job type JobType.objects.edit_job_type( job_type_id=job_type_id, interface=interface, trigger_rule=trigger_rule, remove_trigger_rule=remove_trigger_rule, error_mapping=error_mapping, custom_resources=custom_resources, configuration=configuration, secrets=secrets, **extra_fields) except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition, InvalidSecretsConfiguration, ValueError) as ex: logger.exception('Unable to update job type: %i', job_type_id) raise BadParameter(unicode(ex)) # Fetch the full job type with details try: job_type = JobType.objects.get_details(job_type.id) except JobType.DoesNotExist: raise Http404 serializer = self.get_serializer(job_type) return Response(serializer.data)