def get_job_types(recipe_types=None,
job_type_ids=None,
job_type_names=None,
job_type_categories=None):
"""Exports all the job types in the system based on the given filters.
:param recipe_types: Only include job types that are referenced by the given recipe types.
:type recipe_types: list[:class:`recipe.models.RecipeType`]
:param job_type_ids: A list of unique job type identifiers to include.
:type job_type_ids: list[str]
:param job_type_names: A list of job type system names to include.
:type job_type_names: list[str]
:param job_type_categories: A list of job type category names to include.
:type job_type_categories: list[str]
:returns: A list of matching job types.
:rtype: list[:class:`job.models.JobType`]
"""
# Build a set of job type keys referenced by the recipe types
job_type_keys = set()
if recipe_types and not (job_type_ids or job_type_names
or job_type_categories):
for recipe_type in recipe_types:
job_type_keys.update(
recipe_type.get_recipe_definition().get_job_type_keys())
if not job_type_keys:
return []
# System job types should never be exported
job_types = JobType.objects.exclude(
category='system').select_related('trigger_rule')
# Filter by the referenced job type keys
job_type_filters = []
for job_type_key in job_type_keys:
job_type_filter = Q(name=job_type_key[0], version=job_type_key[1])
job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter
if job_type_filters:
job_types = job_types.filter(job_type_filters)
# Filter by additional passed arguments
if job_type_ids:
job_types = job_types.filter(id__in=job_type_ids)
if job_type_names:
job_types = job_types.filter(name__in=job_type_names)
if job_type_categories:
job_types = job_types.filter(category__in=job_type_categories)
# Scrub configuration for secrets
for job_type in job_types:
if job_type.configuration:
configuration = JobConfigurationV2(job_type.configuration)
interface = JobInterface(job_type.manifest)
configuration.validate(interface.get_dict())
job_type.configuration = configuration.get_dict()
return job_types
class TestJobTypeManagerEditJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.job_type = JobType.objects.create_job_type('name', '1.0', self.job_interface)
new_valid_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['application/json'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.new_valid_job_interface = JobInterface(new_valid_interface)
new_invalid_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]}
self.new_invalid_job_interface = JobInterface(new_invalid_interface)
self.definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type.name,
'version': self.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}]
}
self.recipe_def = RecipeDefinition(self.definition)
self.recipe = RecipeType.objects.create_recipe_type('name', '1.0', 'title', 'description', self.recipe_def,
None)
def test_valid_interface(self):
"""Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change
is made"""
# Call test
JobType.objects.edit_job_type(self.job_type.id, self.new_valid_job_interface)
# Check results
job_type = JobType.objects.get(pk=self.job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_valid_job_interface.get_dict())
self.assertEqual(job_type.revision_num, 2)
# New revision due to interface change
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 2)
def test_invalid_interface(self):
"""Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface
change is made"""
# Call test
self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type, self.job_type.id,
self.new_invalid_job_interface)
# Check results
job_type = JobType.objects.get(pk=self.job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
class TestJobTypeManagerEditJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.error = error_test_utils.create_error()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
new_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]}
self.new_job_interface = JobInterface(new_interface)
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)
self.new_configuration = {
'version': '1.0',
'condition': {
'media_type': 'application/json'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.new_trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE,
self.new_configuration)
def test_change_general_fields(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to some general fields'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
priority = 12
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
new_title = 'my new title'
new_priority = 13
new_error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-16': self.error.name,
}
})
new_is_paused = True
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title,
priority=priority, error_mapping=error_mapping)
# Call test
JobType.objects.edit_job_type(job_type.id, title=new_title, priority=new_priority,
error_mapping=new_error_mapping, is_paused=new_is_paused)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
self.assertEqual(job_type.title, new_title)
self.assertEqual(job_type.priority, new_priority)
self.assertDictEqual(job_type.get_error_interface().get_dict(), new_error_mapping.get_dict())
self.assertEqual(job_type.is_paused, new_is_paused)
self.assertIsNotNone(job_type.paused)
def test_change_to_interface(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to the interface'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, self.new_job_interface, None, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict())
self.assertEqual(job_type.revision_num, 2)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
# New revision due to interface change
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 2)
def test_change_to_trigger_rule(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to the trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.new_trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, None, new_trigger_rule, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertFalse(trigger_rule.is_active)
new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id)
self.assertTrue(new_trigger_rule.is_active)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def test_remove_trigger_rule(self):
'''Tests calling JobTypeManager.edit_job_type() that removes the trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, None, None, True)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertFalse(trigger_rule.is_active)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def test_change_to_both(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to both the definition and the trigger rule
'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.new_trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, self.new_job_interface, new_trigger_rule, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict())
self.assertEqual(job_type.revision_num, 2)
self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertFalse(trigger_rule.is_active)
new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id)
self.assertTrue(new_trigger_rule.is_active)
# New revision due to definition change
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 2)
def test_invalid_trigger_rule(self):
'''Tests calling JobTypeManager.edit_job_type() with a new invalid trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE,
configuration=self.new_trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
self.assertRaises(InvalidConnection, JobType.objects.edit_job_type, job_type.id, self.new_job_interface,
new_trigger_rule, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def test_system_job_type(self):
'''Tests calling JobTypeManager.edit_job_type() for a system job type'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
new_title = 'my new title'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title)
job_type.is_system = True
job_type.save()
# Call test
self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
# No change
self.assertEqual(job_type.title, title)
def test_uneditable_field(self):
'''Tests calling JobTypeManager.edit_job_type() to change an uneditable field'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
new_title = 'my new title'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title)
# Call test
self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title, is_system=True)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
# No change
self.assertEqual(job_type.title, title)
def test_invalid_error_mapping(self):
'''Tests calling JobTypeManager.edit_job_type() with an invalid error mapping'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'1': 'test-invalid-error',
}
})
# Call test
self.assertRaises(Exception, JobType.objects.edit_job_type, name, version, self.job_interface,
error_mapping=error_mapping, title=title, description=description, priority=priority,
is_system=is_system)
class TestJobTypeManagerCreateJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.error = error_test_utils.create_error()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)
self.error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
def test_successful_no_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() successfully with no trigger rule or error mapping'''
name = 'my-job-type'
version = '1.0'
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
def test_successful_with_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() successfully with a trigger rule and error mapping'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, self.error_mapping)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
self.assertDictEqual(job_type.get_error_interface().get_dict(), self.error_mapping.get_dict())
def test_invalid_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() with an invalid trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE,
configuration=self.trigger_config.get_dict())
# Call test
self.assertRaises(InvalidConnection, JobType.objects.create_job_type, name, version, self.job_interface,
trigger_rule, self.error_mapping)
def test_successful_other_fields(self):
'''Tests calling JobTypeManager.create_job_type() successfully with additional fields'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
description=description, priority=priority)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
self.assertEqual(job_type.description, description)
self.assertEqual(job_type.priority, priority)
self.assertIsNone(job_type.archived)
self.assertIsNone(job_type.paused)
def test_successful_paused(self):
'''Tests calling JobTypeManager.create_job_type() and pausing it'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_paused = True
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
description=description, priority=priority, is_paused=is_paused)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
self.assertEqual(job_type.description, description)
self.assertEqual(job_type.priority, priority)
self.assertEqual(job_type.is_paused, is_paused)
self.assertIsNotNone(job_type.paused)
def test_uneditable_field(self):
'''Tests calling JobTypeManager.create_job_type() with an uneditable field'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
# Call test
self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, title=title,
description=description, priority=priority, is_system=is_system)
def test_invalid_error_mapping(self):
'''Tests calling JobTypeManager.create_job_type() with an invalid error mapping'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'1': 'test-invalid-error',
}
})
# Call test
self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface,
error_mapping=error_mapping, title=title, description=description, priority=priority,
is_system=is_system)
class TestJobTypeManagerEditJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
interface = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]
}
self.job_interface = JobInterface(interface)
self.job_type = JobType.objects.create_job_type(
'name', '1.0', self.job_interface)
new_valid_interface = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['application/json'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]
}
self.new_valid_job_interface = JobInterface(new_valid_interface)
new_invalid_interface = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]
}
self.new_invalid_job_interface = JobInterface(new_invalid_interface)
self.definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type.name,
'version': self.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}]
}
self.recipe_def = RecipeDefinition(self.definition)
self.recipe = RecipeType.objects.create_recipe_type(
'name', '1.0', 'title', 'description', self.recipe_def, None)
def test_valid_interface(self):
"""Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change
is made"""
# Call test
JobType.objects.edit_job_type(self.job_type.id,
self.new_valid_job_interface)
# Check results
job_type = JobType.objects.get(pk=self.job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(),
self.new_valid_job_interface.get_dict())
self.assertEqual(job_type.revision_num, 2)
# New revision due to interface change
num_of_revs = JobTypeRevision.objects.filter(
job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 2)
def test_invalid_interface(self):
"""Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface
change is made"""
# Call test
self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type,
self.job_type.id, self.new_invalid_job_interface)
# Check results
job_type = JobType.objects.get(pk=self.job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(),
self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
num_of_revs = JobTypeRevision.objects.filter(
job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def _import_job_type(job_type_dict, job_type=None, validating=False):
"""Attempts to apply the given job types configuration to the system.
Note that proper model locking must be performed before calling this method.
:param job_type_dict: A dictionary of job type configuration changes to import.
:type job_type_dict: dict
:param job_type: The existing job type model to update if applicable.
:type job_type: :class:`job.models.JobType`
:param validating: Flag to determine if running a validate or commit transaction.
:type validating: bool
:returns: A list of warnings discovered during import.
:rtype: list[:class:`port.schema.ValidationWarning`]
:raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
"""
warnings = []
# Parse the JSON content into validated model fields
job_type_serializer = serializers.ConfigurationJobTypeSerializer(job_type, data=job_type_dict)
if not job_type_serializer.is_valid():
raise InvalidConfiguration('Invalid job type schema: %s -> %s' % (job_type_dict['name'],
job_type_serializer.errors))
result = job_type_serializer.validated_data
# Importing system-level job types is not allowed
if result.get('category') == 'system' or (job_type and job_type.category == 'system'):
raise InvalidConfiguration('System job types cannot be imported: %s' % result.get('name'))
# Validate the job interface
try:
interface_dict = None
if 'interface' in result:
interface_dict = result.get('interface')
elif job_type:
interface_dict = job_type.manifest
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type interface invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the job configuration
try:
configuration_dict = None
secrets = None
if 'configuration' in result:
configuration_dict = result.get('configuration')
elif job_type:
configuration_dict = job_type.configuration
if interface:
configuration = JobConfigurationV2(configuration_dict)
if not validating:
secrets = configuration.get_secret_settings(interface.get_dict())
warnings.extend(configuration.validate(interface.get_dict()))
except InvalidJobConfiguration as ex:
raise InvalidConfiguration('Job type configuration invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the error mapping
try:
error_mapping_dict = None
if 'error_mapping' in result:
error_mapping_dict = result.get('error_mapping')
elif job_type:
error_mapping_dict = job_type.error_mapping
error_mapping = create_legacy_error_mapping(error_mapping_dict)
error_mapping.validate_legacy()
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the trigger rule
trigger_rule = None
if 'trigger_rule' in result and result.get('trigger_rule'):
trigger_rule = TriggerRule(**result.get('trigger_rule'))
if trigger_rule:
trigger_config = trigger_rule.get_configuration()
if not isinstance(trigger_config, JobTriggerRuleConfiguration):
logger.exception('Job type trigger rule type invalid')
raise InvalidConfiguration('Job type trigger type invalid: %s -> %s' % (result.get('name'),
trigger_rule.type))
try:
warnings.extend(trigger_config.validate_trigger_for_job(interface))
# Create a new rule when the trigger content was provided
if job_type_dict.get('trigger_rule'):
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
trigger_rule.is_active)
except (InvalidTriggerType, InvalidTriggerRule, InvalidConnection) as ex:
logger.exception('Job type trigger rule invalid')
raise InvalidConfiguration('Job type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict['trigger_rule']
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping', 'configuration'}
for key in job_type_dict:
if key not in base_fields:
if key in JobType.UNEDITABLE_FIELDS:
warnings.append(ValidationWarning(
'read-only', 'Job type includes read-only field: %s -> %s' % (result.get('name'), key)
))
else:
extra_fields[key] = job_type_dict[key]
# Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
if 'mem_required' in extra_fields:
extra_fields['mem_const_required'] = extra_fields['mem_required']
del extra_fields['mem_required']
# Edit or create the associated job type model
if job_type:
try:
JobType.objects.edit_job_type_v5(job_type.id, interface=interface, trigger_rule=trigger_rule,
remove_trigger_rule=remove_trigger_rule,
error_mapping=error_mapping,
configuration=configuration, secrets=secrets, **extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
InvalidSecretsConfiguration) as ex:
logger.exception('Job type edit failed')
raise InvalidConfiguration(
'Unable to edit existing job type: %s -> %s' % (result.get('name'), unicode(ex)))
else:
try:
JobType.objects.create_job_type_v5(name=result.get('name'), version=result.get('version'),
interface=interface, trigger_rule=trigger_rule,
error_mapping=error_mapping, configuration=configuration,
secrets=secrets, **extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
InvalidSecretsConfiguration) as ex:
logger.exception('Job type create failed')
raise InvalidConfiguration('Unable to create new job type: %s -> %s' % (result.get('name'), unicode(ex)))
return warnings
def create(self, request):
"""Creates a new job type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
# Validate the job interface
interface_dict = rest_util.parse_dict(request, 'interface')
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the job configuration and pull out secrets
configuration_dict = rest_util.parse_dict(request,
'configuration',
required=False)
configuration = None
secrets = None
try:
if configuration_dict:
configuration = JobConfiguration(configuration_dict)
secrets = configuration.get_secret_settings(
interface.get_dict())
configuration.validate(interface.get_dict())
except InvalidJobConfiguration as ex:
raise BadParameter('Job type configuration invalid: %s' %
unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request,
'error_mapping',
required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
error_mapping.validate()
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' %
unicode(ex))
# Validate the custom resources
resources_dict = rest_util.parse_dict(request,
'custom_resources',
required=False)
custom_resources = None
try:
if resources_dict:
custom_resources = Resources(resources_dict)
except InvalidResources as ex:
raise BadParameter('Job type custom resources invalid: %s' %
unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
is_active = trigger_rule_dict[
'is_active'] if 'is_active' in trigger_rule_dict else True
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for new job type: %s',
name)
raise BadParameter(unicode(ex))
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {
'name', 'version', 'interface', 'trigger_rule', 'error_mapping',
'custom_resources', 'configuration'
}
for key, value in request.data.iteritems():
if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
extra_fields[key] = value
# Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
if 'mem_required' in extra_fields:
extra_fields['mem_const_required'] = extra_fields['mem_required']
del extra_fields['mem_required']
try:
with transaction.atomic():
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule_dict['configuration'], name, is_active)
# Create the job type
job_type = JobType.objects.create_job_type(
name=name,
version=version,
interface=interface,
trigger_rule=trigger_rule,
error_mapping=error_mapping,
custom_resources=custom_resources,
configuration=configuration,
secrets=secrets,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
InvalidConnection, InvalidSecretsConfiguration,
ValueError) as ex:
logger.exception('Unable to create new job type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details(job_type.id)
except JobType.DoesNotExist:
raise Http404
url = reverse('job_type_details_view',
args=[job_type.id],
request=request)
serializer = JobTypeDetailsSerializer(job_type)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
def patch(self, request, job_type_id):
"""Edits an existing job type and returns the updated details
:param request: the HTTP PATCH request
:type request: :class:`rest_framework.request.Request`
:param job_type_id: The ID for the job type.
:type job_type_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
# Validate the job interface
interface_dict = rest_util.parse_dict(request,
'interface',
required=False)
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the job configuration and pull out secrets
configuration_dict = rest_util.parse_dict(request,
'configuration',
required=False)
configuration = None
secrets = None
try:
if configuration_dict:
configuration = JobConfiguration(configuration_dict)
if interface:
secrets = configuration.get_secret_settings(
interface.get_dict())
configuration.validate(interface.get_dict())
else:
stored_interface = JobType.objects.values_list(
'interface', flat=True).get(pk=job_type_id)
secrets = configuration.get_secret_settings(
stored_interface)
configuration.validate(stored_interface)
except InvalidJobConfiguration as ex:
raise BadParameter('Job type configuration invalid: %s' %
unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request,
'error_mapping',
required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
error_mapping.validate()
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' %
unicode(ex))
# Validate the custom resources
resources_dict = rest_util.parse_dict(request,
'custom_resources',
required=False)
custom_resources = None
try:
if resources_dict:
custom_resources = Resources(resources_dict)
except InvalidResources as ex:
raise BadParameter('Job type custom resources invalid: %s' %
unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
is_active = trigger_rule_dict[
'is_active'] if 'is_active' in trigger_rule_dict else True
remove_trigger_rule = rest_util.has_params(
request, 'trigger_rule') and not trigger_rule_dict
# Fetch the current job type model
try:
job_type = JobType.objects.select_related('trigger_rule').get(
pk=job_type_id)
except JobType.DoesNotExist:
raise Http404
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for job type: %i',
job_type_id)
raise BadParameter(unicode(ex))
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {
'name', 'version', 'interface', 'trigger_rule', 'error_mapping',
'custom_resources', 'configuration'
}
for key, value in request.data.iteritems():
if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
extra_fields[key] = value
# Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
if 'mem_required' in extra_fields:
extra_fields['mem_const_required'] = extra_fields['mem_required']
del extra_fields['mem_required']
try:
from recipe.configuration.definition.exceptions import InvalidDefinition
except:
logger.exception(
'Failed to import higher level recipe application.')
pass
try:
with transaction.atomic():
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule_dict['configuration'], job_type.name,
is_active)
# Update the active state separately if that is only given trigger field
if not trigger_rule and job_type.trigger_rule and 'is_active' in trigger_rule_dict:
job_type.trigger_rule.is_active = is_active
job_type.trigger_rule.save()
# Edit the job type
JobType.objects.edit_job_type(
job_type_id=job_type_id,
interface=interface,
trigger_rule=trigger_rule,
remove_trigger_rule=remove_trigger_rule,
error_mapping=error_mapping,
custom_resources=custom_resources,
configuration=configuration,
secrets=secrets,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
InvalidConnection, InvalidDefinition,
InvalidSecretsConfiguration, ValueError) as ex:
logger.exception('Unable to update job type: %i', job_type_id)
raise BadParameter(unicode(ex))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details(job_type.id)
except JobType.DoesNotExist:
raise Http404
serializer = self.get_serializer(job_type)
return Response(serializer.data)
