Exemple #1
0
def get_job_types(recipe_types=None,
                  job_type_ids=None,
                  job_type_names=None,
                  job_type_categories=None):
    """Exports all the job types in the system based on the given filters.

    :param recipe_types: Only include job types that are referenced by the given recipe types.
    :type recipe_types: list[:class:`recipe.models.RecipeType`]
    :param job_type_ids: A list of unique job type identifiers to include.
    :type job_type_ids: list[str]
    :param job_type_names: A list of job type system names to include.
    :type job_type_names: list[str]
    :param job_type_categories: A list of job type category names to include.
    :type job_type_categories: list[str]
    :returns: A list of matching job types.
    :rtype: list[:class:`job.models.JobType`]
    """

    # Build a set of job type keys referenced by the recipe types
    job_type_keys = set()
    if recipe_types and not (job_type_ids or job_type_names
                             or job_type_categories):
        for recipe_type in recipe_types:
            job_type_keys.update(
                recipe_type.get_recipe_definition().get_job_type_keys())
        if not job_type_keys:
            return []

    # System job types should never be exported
    job_types = JobType.objects.exclude(
        category='system').select_related('trigger_rule')

    # Filter by the referenced job type keys
    job_type_filters = []
    for job_type_key in job_type_keys:
        job_type_filter = Q(name=job_type_key[0], version=job_type_key[1])
        job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter
    if job_type_filters:
        job_types = job_types.filter(job_type_filters)

    # Filter by additional passed arguments
    if job_type_ids:
        job_types = job_types.filter(id__in=job_type_ids)
    if job_type_names:
        job_types = job_types.filter(name__in=job_type_names)
    if job_type_categories:
        job_types = job_types.filter(category__in=job_type_categories)

    # Scrub configuration for secrets
    for job_type in job_types:
        if job_type.configuration:
            configuration = JobConfigurationV2(job_type.configuration)
            interface = JobInterface(job_type.manifest)
            configuration.validate(interface.get_dict())
            job_type.configuration = configuration.get_dict()

    return job_types
Exemple #2
0
class TestJobTypeManagerEditJobType(TransactionTestCase):

    def setUp(self):
        django.setup()

        self.workspace = storage_test_utils.create_workspace()

        interface = {
            'version': '1.0',
            'command': 'my_command',
            'command_arguments': 'args',
            'input_data': [{
                'name': 'Test Input 1',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'output_data': [{
                'name': 'Test Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]}
        self.job_interface = JobInterface(interface)
        self.job_type = JobType.objects.create_job_type('name', '1.0', self.job_interface)

        new_valid_interface = {
            'version': '1.0',
            'command': 'my_command',
            'command_arguments': 'args',
            'input_data': [{
                'name': 'Test Input 1',
                'type': 'file',
                'media_types': ['application/json'],
            }],
            'output_data': [{
                'name': 'Test Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]}
        self.new_valid_job_interface = JobInterface(new_valid_interface)

        new_invalid_interface = {
            'version': '1.0',
            'command': 'my_command',
            'command_arguments': 'args',
            'input_data': [{
                'name': 'Test Input 2',
                'type': 'files',
                'media_types': ['image/png', 'image/tiff'],
            }],
            'output_data': [{
                'name': 'Test Output 2',
                'type': 'file',
            }]}
        self.new_invalid_job_interface = JobInterface(new_invalid_interface)

        self.definition = {
            'version': '1.0',
            'input_data': [{
                'name': 'Recipe Input',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'jobs': [{
                'name': 'Job 1',
                'job_type': {
                    'name': self.job_type.name,
                    'version': self.job_type.version,
                },
                'recipe_inputs': [{
                    'recipe_input': 'Recipe Input',
                    'job_input': 'Test Input 1',
                }]
            }]
        }
        self.recipe_def = RecipeDefinition(self.definition)
        self.recipe = RecipeType.objects.create_recipe_type('name', '1.0', 'title', 'description', self.recipe_def,
                                                            None)

    def test_valid_interface(self):
        """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change
        is made"""

        # Call test
        JobType.objects.edit_job_type(self.job_type.id, self.new_valid_job_interface)

        # Check results
        job_type = JobType.objects.get(pk=self.job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_valid_job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 2)
        # New revision due to interface change
        num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 2)

    def test_invalid_interface(self):
        """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface
        change is made"""

        # Call test
        self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type, self.job_type.id,
                          self.new_invalid_job_interface)

        # Check results
        job_type = JobType.objects.get(pk=self.job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 1)
Exemple #3
0
class TestJobTypeManagerEditJobType(TransactionTestCase):

    def setUp(self):
        django.setup()

        self.workspace = storage_test_utils.create_workspace()
        self.error = error_test_utils.create_error()

        interface = {
            'version': '1.0',
            'command': 'my_command',
            'command_arguments': 'args',
            'input_data': [{
                'name': 'Test Input 1',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'output_data': [{
                'name': 'Test Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]}
        self.job_interface = JobInterface(interface)

        new_interface = {
            'version': '1.0',
            'command': 'my_command',
            'command_arguments': 'args',
            'input_data': [{
                'name': 'Test Input 2',
                'type': 'files',
                'media_types': ['image/png', 'image/tiff'],
            }],
            'output_data': [{
                'name': 'Test Output 2',
                'type': 'file',
            }]}
        self.new_job_interface = JobInterface(new_interface)

        self.configuration = {
            'version': '1.0',
            'condition': {
                'media_type': 'text/plain'
            },
            'data': {
                'input_data_name': 'Test Input 1',
                'workspace_name': self.workspace.name
            }
        }
        self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)

        self.new_configuration = {
            'version': '1.0',
            'condition': {
                'media_type': 'application/json'
            },
            'data': {
                'input_data_name': 'Test Input 1',
                'workspace_name': self.workspace.name
            }
        }
        self.new_trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE,
                                                                              self.new_configuration)

    def test_change_general_fields(self):
        '''Tests calling JobTypeManager.edit_job_type() with a change to some general fields'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        priority = 12
        error_mapping = ErrorInterface({
            'version': '1.0',
            'exit_codes': {
                '-15': self.error.name,
            }
        })
        new_title = 'my new title'
        new_priority = 13
        new_error_mapping = ErrorInterface({
            'version': '1.0',
            'exit_codes': {
                '-16': self.error.name,
            }
        })
        new_is_paused = True
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title,
                                                   priority=priority, error_mapping=error_mapping)

        # Call test
        JobType.objects.edit_job_type(job_type.id, title=new_title, priority=new_priority,
                                      error_mapping=new_error_mapping, is_paused=new_is_paused)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
        trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
        self.assertTrue(trigger_rule.is_active)
        self.assertEqual(job_type.title, new_title)
        self.assertEqual(job_type.priority, new_priority)
        self.assertDictEqual(job_type.get_error_interface().get_dict(), new_error_mapping.get_dict())
        self.assertEqual(job_type.is_paused, new_is_paused)
        self.assertIsNotNone(job_type.paused)

    def test_change_to_interface(self):
        '''Tests calling JobTypeManager.edit_job_type() with a change to the interface'''

        name = 'my-job-type'
        version = '1.0'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)

        # Call test
        JobType.objects.edit_job_type(job_type.id, self.new_job_interface, None, False)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 2)
        self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
        trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
        self.assertTrue(trigger_rule.is_active)
        # New revision due to interface change
        num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 2)

    def test_change_to_trigger_rule(self):
        '''Tests calling JobTypeManager.edit_job_type() with a change to the trigger rule'''

        name = 'my-job-type'
        version = '1.0'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                                  configuration=self.new_trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)

        # Call test
        JobType.objects.edit_job_type(job_type.id, None, new_trigger_rule, False)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id)
        trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
        self.assertFalse(trigger_rule.is_active)
        new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id)
        self.assertTrue(new_trigger_rule.is_active)
        num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 1)

    def test_remove_trigger_rule(self):
        '''Tests calling JobTypeManager.edit_job_type() that removes the trigger rule'''

        name = 'my-job-type'
        version = '1.0'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)

        # Call test
        JobType.objects.edit_job_type(job_type.id, None, None, True)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertIsNone(job_type.trigger_rule)
        trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
        self.assertFalse(trigger_rule.is_active)
        num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 1)

    def test_change_to_both(self):
        '''Tests calling JobTypeManager.edit_job_type() with a change to both the definition and the trigger rule
        '''

        name = 'my-job-type'
        version = '1.0'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                                  configuration=self.new_trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)

        # Call test
        JobType.objects.edit_job_type(job_type.id, self.new_job_interface, new_trigger_rule, False)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 2)
        self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id)
        trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
        self.assertFalse(trigger_rule.is_active)
        new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id)
        self.assertTrue(new_trigger_rule.is_active)
        # New revision due to definition change
        num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 2)

    def test_invalid_trigger_rule(self):
        '''Tests calling JobTypeManager.edit_job_type() with a new invalid trigger rule'''

        name = 'my-job-type'
        version = '1.0'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE,
                                                                  configuration=self.new_trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
        
        # Call test
        self.assertRaises(InvalidConnection, JobType.objects.edit_job_type, job_type.id, self.new_job_interface,
                          new_trigger_rule, False)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
        trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
        self.assertTrue(trigger_rule.is_active)
        num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 1)

    def test_system_job_type(self):
        '''Tests calling JobTypeManager.edit_job_type() for a system job type'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        new_title = 'my new title'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title)
        job_type.is_system = True
        job_type.save()
        
        # Call test
        self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        # No change
        self.assertEqual(job_type.title, title)

    def test_uneditable_field(self):
        '''Tests calling JobTypeManager.edit_job_type() to change an uneditable field'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        new_title = 'my new title'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title)

        # Call test
        self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title, is_system=True)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        # No change
        self.assertEqual(job_type.title, title)

    def test_invalid_error_mapping(self):
        '''Tests calling JobTypeManager.edit_job_type() with an invalid error mapping'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        description = 'my-description'
        priority = 13
        is_system = True
        error_mapping = ErrorInterface({
            'version': '1.0',
            'exit_codes': {
                '1': 'test-invalid-error',
            }
        })

        # Call test
        self.assertRaises(Exception, JobType.objects.edit_job_type, name, version, self.job_interface,
                          error_mapping=error_mapping, title=title, description=description, priority=priority,
                          is_system=is_system)
Exemple #4
0
class TestJobTypeManagerCreateJobType(TransactionTestCase):

    def setUp(self):
        django.setup()

        self.workspace = storage_test_utils.create_workspace()
        self.error = error_test_utils.create_error()

        interface = {
            'version': '1.0',
            'command': 'my_command',
            'command_arguments': 'args',
            'input_data': [{
                'name': 'Test Input 1',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'output_data': [{
                'name': 'Test Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]}
        self.job_interface = JobInterface(interface)

        self.configuration = {
            'version': '1.0',
            'condition': {
                'media_type': 'text/plain'
            },
            'data': {
                'input_data_name': 'Test Input 1',
                'workspace_name': self.workspace.name
            }
        }
        self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)

        self.error_mapping = ErrorInterface({
            'version': '1.0',
            'exit_codes': {
                '-15': self.error.name,
            }
        })

    def test_successful_no_trigger_rule(self):
        '''Tests calling JobTypeManager.create_job_type() successfully with no trigger rule or error mapping'''

        name = 'my-job-type'
        version = '1.0'

        # Call test
        job_type = JobType.objects.create_job_type(name, version, self.job_interface)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertIsNone(job_type.trigger_rule_id)
        self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())

    def test_successful_with_trigger_rule(self):
        '''Tests calling JobTypeManager.create_job_type() successfully with a trigger rule and error mapping'''

        name = 'my-job-type'
        version = '1.0'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
                                                              configuration=self.trigger_config.get_dict())

        # Call test
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, self.error_mapping)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
        trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
        self.assertTrue(trigger_rule.is_active)
        self.assertDictEqual(job_type.get_error_interface().get_dict(), self.error_mapping.get_dict())

    def test_invalid_trigger_rule(self):
        '''Tests calling JobTypeManager.create_job_type() with an invalid trigger rule'''

        name = 'my-job-type'
        version = '1.0'
        trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE,
                                                              configuration=self.trigger_config.get_dict())

        # Call test
        self.assertRaises(InvalidConnection, JobType.objects.create_job_type, name, version, self.job_interface,
                          trigger_rule, self.error_mapping)

    def test_successful_other_fields(self):
        '''Tests calling JobTypeManager.create_job_type() successfully with additional fields'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        description = 'my-description'
        priority = 13

        # Call test
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
                                                   description=description, priority=priority)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertIsNone(job_type.trigger_rule_id)
        self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
        self.assertEqual(job_type.description, description)
        self.assertEqual(job_type.priority, priority)
        self.assertIsNone(job_type.archived)
        self.assertIsNone(job_type.paused)

    def test_successful_paused(self):
        '''Tests calling JobTypeManager.create_job_type() and pausing it'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        description = 'my-description'
        priority = 13
        is_paused = True

        # Call test
        job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
                                                   description=description, priority=priority, is_paused=is_paused)

        # Check results
        job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        self.assertIsNone(job_type.trigger_rule_id)
        self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
        self.assertEqual(job_type.description, description)
        self.assertEqual(job_type.priority, priority)
        self.assertEqual(job_type.is_paused, is_paused)
        self.assertIsNotNone(job_type.paused)

    def test_uneditable_field(self):
        '''Tests calling JobTypeManager.create_job_type() with an uneditable field'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        description = 'my-description'
        priority = 13
        is_system = True

        # Call test
        self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, title=title,
                          description=description, priority=priority, is_system=is_system)

    def test_invalid_error_mapping(self):
        '''Tests calling JobTypeManager.create_job_type() with an invalid error mapping'''

        name = 'my-job-type'
        version = '1.0'
        title = 'my title'
        description = 'my-description'
        priority = 13
        is_system = True
        error_mapping = ErrorInterface({
            'version': '1.0',
            'exit_codes': {
                '1': 'test-invalid-error',
            }
        })

        # Call test
        self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface,
                          error_mapping=error_mapping, title=title, description=description, priority=priority,
                          is_system=is_system)
Exemple #5
0
class TestJobTypeManagerEditJobType(TransactionTestCase):
    def setUp(self):
        django.setup()

        self.workspace = storage_test_utils.create_workspace()

        interface = {
            'version':
            '1.0',
            'command':
            'my_command',
            'command_arguments':
            'args',
            'input_data': [{
                'name': 'Test Input 1',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'output_data': [{
                'name': 'Test Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]
        }
        self.job_interface = JobInterface(interface)
        self.job_type = JobType.objects.create_job_type(
            'name', '1.0', self.job_interface)

        new_valid_interface = {
            'version':
            '1.0',
            'command':
            'my_command',
            'command_arguments':
            'args',
            'input_data': [{
                'name': 'Test Input 1',
                'type': 'file',
                'media_types': ['application/json'],
            }],
            'output_data': [{
                'name': 'Test Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]
        }
        self.new_valid_job_interface = JobInterface(new_valid_interface)

        new_invalid_interface = {
            'version':
            '1.0',
            'command':
            'my_command',
            'command_arguments':
            'args',
            'input_data': [{
                'name': 'Test Input 2',
                'type': 'files',
                'media_types': ['image/png', 'image/tiff'],
            }],
            'output_data': [{
                'name': 'Test Output 2',
                'type': 'file',
            }]
        }
        self.new_invalid_job_interface = JobInterface(new_invalid_interface)

        self.definition = {
            'version':
            '1.0',
            'input_data': [{
                'name': 'Recipe Input',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'jobs': [{
                'name':
                'Job 1',
                'job_type': {
                    'name': self.job_type.name,
                    'version': self.job_type.version,
                },
                'recipe_inputs': [{
                    'recipe_input': 'Recipe Input',
                    'job_input': 'Test Input 1',
                }]
            }]
        }
        self.recipe_def = RecipeDefinition(self.definition)
        self.recipe = RecipeType.objects.create_recipe_type(
            'name', '1.0', 'title', 'description', self.recipe_def, None)

    def test_valid_interface(self):
        """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change
        is made"""

        # Call test
        JobType.objects.edit_job_type(self.job_type.id,
                                      self.new_valid_job_interface)

        # Check results
        job_type = JobType.objects.get(pk=self.job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(),
                             self.new_valid_job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 2)
        # New revision due to interface change
        num_of_revs = JobTypeRevision.objects.filter(
            job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 2)

    def test_invalid_interface(self):
        """Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface
        change is made"""

        # Call test
        self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type,
                          self.job_type.id, self.new_invalid_job_interface)

        # Check results
        job_type = JobType.objects.get(pk=self.job_type.id)
        self.assertDictEqual(job_type.get_job_interface().get_dict(),
                             self.job_interface.get_dict())
        self.assertEqual(job_type.revision_num, 1)
        num_of_revs = JobTypeRevision.objects.filter(
            job_type_id=job_type.id).count()
        self.assertEqual(num_of_revs, 1)
Exemple #6
0
def _import_job_type(job_type_dict, job_type=None, validating=False):
    """Attempts to apply the given job types configuration to the system.

    Note that proper model locking must be performed before calling this method.

    :param job_type_dict: A dictionary of job type configuration changes to import.
    :type job_type_dict: dict
    :param job_type: The existing job type model to update if applicable.
    :type job_type: :class:`job.models.JobType`
    :param validating: Flag to determine if running a validate or commit transaction.
    :type validating: bool
    :returns: A list of warnings discovered during import.
    :rtype: list[:class:`port.schema.ValidationWarning`]

    :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
    """
    warnings = []

    # Parse the JSON content into validated model fields
    job_type_serializer = serializers.ConfigurationJobTypeSerializer(job_type, data=job_type_dict)
    if not job_type_serializer.is_valid():
        raise InvalidConfiguration('Invalid job type schema: %s -> %s' % (job_type_dict['name'],
                                                                          job_type_serializer.errors))
    result = job_type_serializer.validated_data

    # Importing system-level job types is not allowed
    if result.get('category') == 'system' or (job_type and job_type.category == 'system'):
        raise InvalidConfiguration('System job types cannot be imported: %s' % result.get('name'))

    # Validate the job interface
    try:
        interface_dict = None
        if 'interface' in result:
            interface_dict = result.get('interface')
        elif job_type:
            interface_dict = job_type.manifest
        interface = JobInterface(interface_dict)
    except InvalidInterfaceDefinition as ex:
        raise InvalidConfiguration('Job type interface invalid: %s -> %s' % (result.get('name'), unicode(ex)))

    # Validate the job configuration
    try:
        configuration_dict = None
        secrets = None
        if 'configuration' in result:
            configuration_dict = result.get('configuration')
        elif job_type:
            configuration_dict = job_type.configuration
        if interface:
            configuration = JobConfigurationV2(configuration_dict)
            if not validating:
                secrets = configuration.get_secret_settings(interface.get_dict())
            warnings.extend(configuration.validate(interface.get_dict()))
    except InvalidJobConfiguration as ex:
        raise InvalidConfiguration('Job type configuration invalid: %s -> %s' % (result.get('name'), unicode(ex)))

    # Validate the error mapping
    try:
        error_mapping_dict = None
        if 'error_mapping' in result:
            error_mapping_dict = result.get('error_mapping')
        elif job_type:
            error_mapping_dict = job_type.error_mapping
        error_mapping = create_legacy_error_mapping(error_mapping_dict)
        error_mapping.validate_legacy()
    except InvalidInterfaceDefinition as ex:
        raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' % (result.get('name'), unicode(ex)))

    # Validate the trigger rule
    trigger_rule = None
    if 'trigger_rule' in result and result.get('trigger_rule'):
        trigger_rule = TriggerRule(**result.get('trigger_rule'))
    if trigger_rule:
        trigger_config = trigger_rule.get_configuration()
        if not isinstance(trigger_config, JobTriggerRuleConfiguration):
            logger.exception('Job type trigger rule type invalid')
            raise InvalidConfiguration('Job type trigger type invalid: %s -> %s' % (result.get('name'),
                                                                                    trigger_rule.type))

        try:
            warnings.extend(trigger_config.validate_trigger_for_job(interface))

            # Create a new rule when the trigger content was provided
            if job_type_dict.get('trigger_rule'):
                rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
                trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
                                                                trigger_rule.is_active)
        except (InvalidTriggerType, InvalidTriggerRule, InvalidConnection) as ex:
            logger.exception('Job type trigger rule invalid')
            raise InvalidConfiguration('Job type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
    remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict['trigger_rule']

    # Extract the fields that should be updated as keyword arguments
    extra_fields = {}
    base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping', 'configuration'}
    for key in job_type_dict:
        if key not in base_fields:
            if key in JobType.UNEDITABLE_FIELDS:
                warnings.append(ValidationWarning(
                    'read-only', 'Job type includes read-only field: %s -> %s' % (result.get('name'), key)
                ))
            else:
                extra_fields[key] = job_type_dict[key]
    # Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
    if 'mem_required' in extra_fields:
        extra_fields['mem_const_required'] = extra_fields['mem_required']
        del extra_fields['mem_required']

    # Edit or create the associated job type model
    if job_type:
        try:
            JobType.objects.edit_job_type_v5(job_type.id, interface=interface, trigger_rule=trigger_rule,
                                             remove_trigger_rule=remove_trigger_rule,
                                             error_mapping=error_mapping,
                                             configuration=configuration, secrets=secrets, **extra_fields)
        except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
                InvalidSecretsConfiguration) as ex:
            logger.exception('Job type edit failed')
            raise InvalidConfiguration(
                'Unable to edit existing job type: %s -> %s' % (result.get('name'), unicode(ex)))
    else:
        try:
            JobType.objects.create_job_type_v5(name=result.get('name'), version=result.get('version'),
                                               interface=interface, trigger_rule=trigger_rule,
                                               error_mapping=error_mapping, configuration=configuration,
                                               secrets=secrets, **extra_fields)
        except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
                InvalidSecretsConfiguration) as ex:
            logger.exception('Job type create failed')
            raise InvalidConfiguration('Unable to create new job type: %s -> %s' % (result.get('name'), unicode(ex)))
    return warnings
Exemple #7
0
    def create(self, request):
        """Creates a new job type and returns a link to the detail URL

        :param request: the HTTP POST request
        :type request: :class:`rest_framework.request.Request`
        :rtype: :class:`rest_framework.response.Response`
        :returns: the HTTP response to send back to the user
        """
        name = rest_util.parse_string(request, 'name')
        version = rest_util.parse_string(request, 'version')

        # Validate the job interface
        interface_dict = rest_util.parse_dict(request, 'interface')
        interface = None
        try:
            if interface_dict:
                interface = JobInterface(interface_dict)
        except InvalidInterfaceDefinition as ex:
            raise BadParameter('Job type interface invalid: %s' % unicode(ex))

        # Validate the job configuration and pull out secrets
        configuration_dict = rest_util.parse_dict(request,
                                                  'configuration',
                                                  required=False)
        configuration = None
        secrets = None
        try:
            if configuration_dict:
                configuration = JobConfiguration(configuration_dict)
                secrets = configuration.get_secret_settings(
                    interface.get_dict())
                configuration.validate(interface.get_dict())
        except InvalidJobConfiguration as ex:
            raise BadParameter('Job type configuration invalid: %s' %
                               unicode(ex))

        # Validate the error mapping
        error_dict = rest_util.parse_dict(request,
                                          'error_mapping',
                                          required=False)
        error_mapping = None
        try:
            if error_dict:
                error_mapping = ErrorInterface(error_dict)
                error_mapping.validate()
        except InvalidInterfaceDefinition as ex:
            raise BadParameter('Job type error mapping invalid: %s' %
                               unicode(ex))

        # Validate the custom resources
        resources_dict = rest_util.parse_dict(request,
                                              'custom_resources',
                                              required=False)
        custom_resources = None
        try:
            if resources_dict:
                custom_resources = Resources(resources_dict)
        except InvalidResources as ex:
            raise BadParameter('Job type custom resources invalid: %s' %
                               unicode(ex))

        # Check for optional trigger rule parameters
        trigger_rule_dict = rest_util.parse_dict(request,
                                                 'trigger_rule',
                                                 required=False)
        if (('type' in trigger_rule_dict
             and 'configuration' not in trigger_rule_dict)
                or ('type' not in trigger_rule_dict
                    and 'configuration' in trigger_rule_dict)):
            raise BadParameter(
                'Trigger type and configuration are required together.')
        is_active = trigger_rule_dict[
            'is_active'] if 'is_active' in trigger_rule_dict else True

        # Attempt to look up the trigger handler for the type
        rule_handler = None
        if trigger_rule_dict and 'type' in trigger_rule_dict:
            try:
                rule_handler = trigger_handler.get_trigger_rule_handler(
                    trigger_rule_dict['type'])
            except InvalidTriggerType as ex:
                logger.exception('Invalid trigger type for new job type: %s',
                                 name)
                raise BadParameter(unicode(ex))

        # Extract the fields that should be updated as keyword arguments
        extra_fields = {}
        base_fields = {
            'name', 'version', 'interface', 'trigger_rule', 'error_mapping',
            'custom_resources', 'configuration'
        }
        for key, value in request.data.iteritems():
            if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
                extra_fields[key] = value
        # Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
        if 'mem_required' in extra_fields:
            extra_fields['mem_const_required'] = extra_fields['mem_required']
            del extra_fields['mem_required']

        try:
            with transaction.atomic():

                # Attempt to create the trigger rule
                trigger_rule = None
                if rule_handler and 'configuration' in trigger_rule_dict:
                    trigger_rule = rule_handler.create_trigger_rule(
                        trigger_rule_dict['configuration'], name, is_active)
                # Create the job type
                job_type = JobType.objects.create_job_type(
                    name=name,
                    version=version,
                    interface=interface,
                    trigger_rule=trigger_rule,
                    error_mapping=error_mapping,
                    custom_resources=custom_resources,
                    configuration=configuration,
                    secrets=secrets,
                    **extra_fields)

        except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
                InvalidConnection, InvalidSecretsConfiguration,
                ValueError) as ex:
            logger.exception('Unable to create new job type: %s', name)
            raise BadParameter(unicode(ex))

        # Fetch the full job type with details
        try:
            job_type = JobType.objects.get_details(job_type.id)
        except JobType.DoesNotExist:
            raise Http404

        url = reverse('job_type_details_view',
                      args=[job_type.id],
                      request=request)
        serializer = JobTypeDetailsSerializer(job_type)
        return Response(serializer.data,
                        status=status.HTTP_201_CREATED,
                        headers=dict(location=url))
Exemple #8
0
    def patch(self, request, job_type_id):
        """Edits an existing job type and returns the updated details

        :param request: the HTTP PATCH request
        :type request: :class:`rest_framework.request.Request`
        :param job_type_id: The ID for the job type.
        :type job_type_id: int encoded as a str
        :rtype: :class:`rest_framework.response.Response`
        :returns: the HTTP response to send back to the user
        """

        # Validate the job interface
        interface_dict = rest_util.parse_dict(request,
                                              'interface',
                                              required=False)
        interface = None
        try:
            if interface_dict:
                interface = JobInterface(interface_dict)
        except InvalidInterfaceDefinition as ex:
            raise BadParameter('Job type interface invalid: %s' % unicode(ex))

        # Validate the job configuration and pull out secrets
        configuration_dict = rest_util.parse_dict(request,
                                                  'configuration',
                                                  required=False)
        configuration = None
        secrets = None
        try:
            if configuration_dict:
                configuration = JobConfiguration(configuration_dict)
                if interface:
                    secrets = configuration.get_secret_settings(
                        interface.get_dict())
                    configuration.validate(interface.get_dict())
                else:
                    stored_interface = JobType.objects.values_list(
                        'interface', flat=True).get(pk=job_type_id)
                    secrets = configuration.get_secret_settings(
                        stored_interface)
                    configuration.validate(stored_interface)
        except InvalidJobConfiguration as ex:
            raise BadParameter('Job type configuration invalid: %s' %
                               unicode(ex))

        # Validate the error mapping
        error_dict = rest_util.parse_dict(request,
                                          'error_mapping',
                                          required=False)
        error_mapping = None
        try:
            if error_dict:
                error_mapping = ErrorInterface(error_dict)
                error_mapping.validate()
        except InvalidInterfaceDefinition as ex:
            raise BadParameter('Job type error mapping invalid: %s' %
                               unicode(ex))

        # Validate the custom resources
        resources_dict = rest_util.parse_dict(request,
                                              'custom_resources',
                                              required=False)
        custom_resources = None
        try:
            if resources_dict:
                custom_resources = Resources(resources_dict)
        except InvalidResources as ex:
            raise BadParameter('Job type custom resources invalid: %s' %
                               unicode(ex))

        # Check for optional trigger rule parameters
        trigger_rule_dict = rest_util.parse_dict(request,
                                                 'trigger_rule',
                                                 required=False)
        if (('type' in trigger_rule_dict
             and 'configuration' not in trigger_rule_dict)
                or ('type' not in trigger_rule_dict
                    and 'configuration' in trigger_rule_dict)):
            raise BadParameter(
                'Trigger type and configuration are required together.')
        is_active = trigger_rule_dict[
            'is_active'] if 'is_active' in trigger_rule_dict else True
        remove_trigger_rule = rest_util.has_params(
            request, 'trigger_rule') and not trigger_rule_dict

        # Fetch the current job type model
        try:
            job_type = JobType.objects.select_related('trigger_rule').get(
                pk=job_type_id)
        except JobType.DoesNotExist:
            raise Http404

        # Attempt to look up the trigger handler for the type
        rule_handler = None
        if trigger_rule_dict and 'type' in trigger_rule_dict:
            try:
                rule_handler = trigger_handler.get_trigger_rule_handler(
                    trigger_rule_dict['type'])
            except InvalidTriggerType as ex:
                logger.exception('Invalid trigger type for job type: %i',
                                 job_type_id)
                raise BadParameter(unicode(ex))

        # Extract the fields that should be updated as keyword arguments
        extra_fields = {}
        base_fields = {
            'name', 'version', 'interface', 'trigger_rule', 'error_mapping',
            'custom_resources', 'configuration'
        }
        for key, value in request.data.iteritems():
            if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
                extra_fields[key] = value
        # Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
        if 'mem_required' in extra_fields:
            extra_fields['mem_const_required'] = extra_fields['mem_required']
            del extra_fields['mem_required']

        try:
            from recipe.configuration.definition.exceptions import InvalidDefinition
        except:
            logger.exception(
                'Failed to import higher level recipe application.')
            pass

        try:
            with transaction.atomic():

                # Attempt to create the trigger rule
                trigger_rule = None
                if rule_handler and 'configuration' in trigger_rule_dict:
                    trigger_rule = rule_handler.create_trigger_rule(
                        trigger_rule_dict['configuration'], job_type.name,
                        is_active)

                # Update the active state separately if that is only given trigger field
                if not trigger_rule and job_type.trigger_rule and 'is_active' in trigger_rule_dict:
                    job_type.trigger_rule.is_active = is_active
                    job_type.trigger_rule.save()

                # Edit the job type
                JobType.objects.edit_job_type(
                    job_type_id=job_type_id,
                    interface=interface,
                    trigger_rule=trigger_rule,
                    remove_trigger_rule=remove_trigger_rule,
                    error_mapping=error_mapping,
                    custom_resources=custom_resources,
                    configuration=configuration,
                    secrets=secrets,
                    **extra_fields)
        except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
                InvalidConnection, InvalidDefinition,
                InvalidSecretsConfiguration, ValueError) as ex:
            logger.exception('Unable to update job type: %i', job_type_id)
            raise BadParameter(unicode(ex))

        # Fetch the full job type with details
        try:
            job_type = JobType.objects.get_details(job_type.id)
        except JobType.DoesNotExist:
            raise Http404

        serializer = self.get_serializer(job_type)
        return Response(serializer.data)