def test_get_error_names_empty(self):
"""Tests getting error names when there are no exit codes defined."""
error_interface_dict = {u"version": u"1.0", u"exit_codes": {}}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, set())
def test_get_error_names(self):
"""Tests getting error names from the mapping."""
error_interface_dict = {u"version": u"1.0", u"exit_codes": {u"1": self.error_1.name, u"2": self.error_2.name}}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, {self.error_1.name, self.error_2.name})
def test_get_error_names_none(self):
"""Tests getting error names when there is no error interface."""
error_interface_dict = {}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, set())
def test_get_error_names_none(self):
"""Tests getting error names when there is no error interface."""
error_interface_dict = {}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, set())
def test_get_error_names_unique(self):
"""Tests getting error names without duplicates."""
error_interface_dict = {
u"version": u"1.0",
u"exit_codes": {u"0": self.error_1.name, u"2": self.error_2.name, u"5": self.error_1.name},
}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, {self.error_1.name, self.error_2.name})
def test_validate_empty(self):
"""Tests validating no error names."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {},
}
error_interface = ErrorInterface(error_interface_dict)
# No exception is passing
error_interface.validate()
def test_get_error_zero(self):
""" Tests that no error is returned when the exit_code is 0"""
error_interface_dict = {
u"version": u"1.0",
u"exit_codes": {u"1": self.error_1.name, u"2": self.error_2.name, u"3": self.error_3.name},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(0)
self.assertIsNone(error)
def test_get_error_names_empty(self):
"""Tests getting error names when there are no exit codes defined."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {},
}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, set())
def test_get_error_names_empty(self):
"""Tests getting error names when there are no exit codes defined."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {},
}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, set())
def test_get_error_found(self):
""" Tests that an error model is returned given an exit_code other than 0"""
error_interface_dict = {
u"version": u"1.0",
u"exit_codes": {u"1": self.error_1.name, u"2": self.error_2.name, u"3": self.error_3.name},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(1)
self.assertIsNotNone(error)
self.assertEqual(error.name, Error.objects.get_unknown_error().name)
def test_get_error_missing(self):
""" Tests that an unknown error is returned when a non-registered name is found in the mapping"""
error_interface_dict = {
u"version": u"1.0",
u"exit_codes": {u"1": self.error_1.name, u"2": self.error_2.name, u"3": self.error_3.name},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, Error.objects.get_unknown_error().name)
def test_validate_empty(self):
"""Tests validating no error names."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
},
}
error_interface = ErrorInterface(error_interface_dict)
# No exception is passing
error_interface.validate()
def test_get_error_names(self):
"""Tests getting error names from the mapping."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, {self.error_1.name, self.error_2.name})
def test_get_error_names_unique(self):
"""Tests getting error names without duplicates."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'0': self.error_1.name,
'2': self.error_2.name,
'5': self.error_1.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names, {self.error_1.name, self.error_2.name})
def test_validate_success(self):
"""Tests validating all error names."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
# No exception is passing
error_interface.validate()
def test_get_error_zero(self):
""" Tests that no error is returned when the exit_code is 0"""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(0)
self.assertIsNone(error)
def test_get_error_names(self):
"""Tests getting error names from the mapping."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names,
{self.error_1.name, self.error_2.name})
def test_validate_success(self):
"""Tests validating all error names."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
# No exception is passing
error_interface.validate()
def test_get_error_zero(self):
""" Tests that no error is returned when the exit_code is 0"""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(0)
self.assertIsNone(error)
def test_get_error_missing(self):
''' Tests that an unknown error is returned when a non-registered name is found in the mapping'''
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, Error.objects.get_unknown_error().name)
def test_get_error_missing(self):
"""Tests that general algorithm error is returned when a non-registered name is found in the mapping"""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, 'algorithm-unknown')
def test_get_error_missing(self):
''' Tests that an unknown error is returned when a non-registered name is found in the mapping'''
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, Error.objects.get_unknown_error().name)
def test_get_error_found(self):
""" Tests that an error model is returned given an exit_code other than 0"""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(1)
self.assertIsNotNone(error)
self.assertEqual(error.name, self.error_1.name)
def test_get_error_found(self):
""" Tests that an error model is returned given an exit_code other than 0"""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(1)
self.assertIsNotNone(error)
self.assertEqual(error.name, self.error_1.name)
def test_get_error_names_unique(self):
"""Tests getting error names without duplicates."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'0': self.error_1.name,
'2': self.error_2.name,
'5': self.error_1.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error_names = error_interface.get_error_names()
self.assertSetEqual(error_names,
{self.error_1.name, self.error_2.name})
def test_change_general_fields(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to some general fields'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
priority = 12
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
new_title = 'my new title'
new_priority = 13
new_error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-16': self.error.name,
}
})
new_is_paused = True
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title,
priority=priority, error_mapping=error_mapping)
# Call test
JobType.objects.edit_job_type(job_type.id, title=new_title, priority=new_priority,
error_mapping=new_error_mapping, is_paused=new_is_paused)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
self.assertEqual(job_type.title, new_title)
self.assertEqual(job_type.priority, new_priority)
self.assertDictEqual(job_type.get_error_interface().get_dict(), new_error_mapping.get_dict())
self.assertEqual(job_type.is_paused, new_is_paused)
self.assertIsNotNone(job_type.paused)
def test_get_error_missing(self):
'''Tests that general algorithm error is returned when a non-registered name is found in the mapping'''
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, 'algorithm-unknown')
def test_get_error_missing(self):
"""Tests that general algorithm error is returned when a non-registered name is found in the mapping"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, 'algorithm-unknown')
def test_get_error_missing_default(self):
"""Tests that custom error is returned when a non-registered name is found in the mapping"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
default_error = error_test_utils.create_error()
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4, default_error.name)
self.assertIsNotNone(error)
self.assertEqual(error.name, default_error.name)
def test_get_error_missing_default(self):
"""Tests that custom error is returned when a non-registered name is found in the mapping"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
default_error = error_test_utils.create_error()
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4, default_error.name)
self.assertIsNotNone(error)
self.assertEqual(error.name, default_error.name)
def test_get_error_missing_default(self):
"""Tests that custom error is returned when a non-registered name is found in the mapping"""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
default_error = error_test_utils.create_error()
default_error.is_builtin = True
default_error.save()
# Reset error cache so tests work correctly
reset_error_cache()
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4, default_error.name)
self.assertIsNotNone(error)
self.assertEqual(error.name, default_error.name)
def test_validate_missing(self):
"""Tests validating when some error names are missing."""
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'4': 'test-missing-name',
},
}
error_interface = ErrorInterface(error_interface_dict)
self.assertRaises(InvalidInterfaceDefinition, error_interface.validate)
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.error = error_test_utils.create_error()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)
self.error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
def create(self, request):
"""Creates a new job type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
# Validate the job interface
interface_dict = rest_util.parse_dict(request, 'interface')
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request,
'error_mapping',
required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
error_mapping.validate()
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' %
unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
is_active = trigger_rule_dict[
'is_active'] if 'is_active' in trigger_rule_dict else True
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for new job type: %s',
name)
raise BadParameter(unicode(ex))
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {
'name', 'version', 'interface', 'trigger_rule', 'error_mapping'
}
for key, value in request.data.iteritems():
if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
extra_fields[key] = value
try:
with transaction.atomic():
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule_dict['configuration'], name, is_active)
# Create the job type
job_type = JobType.objects.create_job_type(
name, version, interface, trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
InvalidConnection, ValueError) as ex:
logger.exception('Unable to create new job type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details(job_type.id)
except JobType.DoesNotExist:
raise Http404
url = urlresolvers.reverse('job_type_details_view', args=[job_type.id])
serializer = JobTypeDetailsSerializer(job_type)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
def post(self, request):
"""Validates a new job type and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
# Validate the job interface
interface_dict = rest_util.parse_dict(request, 'interface')
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request,
'error_mapping',
required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' %
unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for job validation: %s',
name)
raise BadParameter(unicode(ex))
# Attempt to look up the trigger rule configuration
trigger_config = None
if rule_handler and 'configuration' in trigger_rule_dict:
try:
trigger_config = rule_handler.create_configuration(
trigger_rule_dict['configuration'])
except InvalidTriggerRule as ex:
logger.exception(
'Invalid trigger rule configuration for job validation: %s',
name)
raise BadParameter(unicode(ex))
try:
from recipe.configuration.definition.exceptions import InvalidDefinition
except:
logger.exception(
'Failed to import higher level recipe application.')
pass
# Validate the job interface
try:
warnings = JobType.objects.validate_job_type(
name, version, interface, error_mapping, trigger_config)
except (InvalidDefinition, InvalidTriggerType,
InvalidTriggerRule) as ex:
logger.exception('Unable to validate new job type: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def patch(self, request, job_type_id):
"""Edits an existing job type and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param job_type_id: The ID for the job type.
:type job_type_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
# Validate the job interface
interface_dict = rest_util.parse_dict(request,
'interface',
required=False)
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request,
'error_mapping',
required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
error_mapping.validate()
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' %
unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
is_active = trigger_rule_dict[
'is_active'] if 'is_active' in trigger_rule_dict else True
remove_trigger_rule = rest_util.has_params(
request, 'trigger_rule') and not trigger_rule_dict
# Fetch the current job type model
try:
job_type = JobType.objects.select_related('trigger_rule').get(
pk=job_type_id)
except JobType.DoesNotExist:
raise Http404
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for job type: %i',
job_type_id)
raise BadParameter(unicode(ex))
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {
'name', 'version', 'interface', 'trigger_rule', 'error_mapping'
}
for key, value in request.data.iteritems():
if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
extra_fields[key] = value
try:
from recipe.configuration.definition.exceptions import InvalidDefinition
except:
logger.exception(
'Failed to import higher level recipe application.')
pass
try:
with transaction.atomic():
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule_dict['configuration'], job_type.name,
is_active)
# Update the active state separately if that is only given trigger field
if not trigger_rule and job_type.trigger_rule and 'is_active' in trigger_rule_dict:
job_type.trigger_rule.is_active = is_active
job_type.trigger_rule.save()
# Edit the job type
JobType.objects.edit_job_type(job_type_id, interface,
trigger_rule,
remove_trigger_rule,
error_mapping, **extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
InvalidConnection, InvalidDefinition, ValueError) as ex:
logger.exception('Unable to update job type: %i', job_type_id)
raise BadParameter(unicode(ex))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details(job_type.id)
except JobType.DoesNotExist:
raise Http404
serializer = self.get_serializer(job_type)
return Response(serializer.data)
def _import_job_type(job_type_dict, job_type=None):
'''Attempts to apply the given job types configuration to the system.
Note that proper model locking must be performed before calling this method.
:param job_type_dict: A dictionary of job type configuration changes to import.
:type job_type_dict: dict
:param job_type: The existing job type model to update if applicable.
:type job_type: :class:`job.models.JobType`
:returns: A list of warnings discovered during import.
:rtype: list[:class:`port.schema.ValidationWarning`]
:raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
'''
warnings = []
# Parse the JSON content into validated model fields
job_type_serializer = serializers.ConfigurationJobTypeSerializer(job_type, data=job_type_dict)
if not job_type_serializer.is_valid():
raise InvalidConfiguration('Invalid job type schema: %s -> %s' % (job_type_dict['name'],
job_type_serializer.errors))
result = job_type_serializer.validated_data
# Importing system-level job types is not allowed
if result.get('category') == 'system' or (job_type and job_type.category == 'system'):
raise InvalidConfiguration('System job types cannot be imported: %s' % result.get('name'))
# Validate the job interface
try:
interface_dict = None
if 'interface' in result:
interface_dict = result.get('interface')
elif job_type:
interface_dict = job_type.interface
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type interface invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the error mapping
try:
error_mapping_dict = None
if 'error_mapping' in result:
error_mapping_dict = result.get('error_mapping')
elif job_type:
error_mapping_dict = job_type.error_mapping
error_mapping = ErrorInterface(error_mapping_dict)
warnings.extend(error_mapping.validate())
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the trigger rule
trigger_rule = None
if 'trigger_rule' in result and result.get('trigger_rule'):
trigger_rule = TriggerRule(**result.get('trigger_rule'))
if trigger_rule:
trigger_config = trigger_rule.get_configuration()
if not isinstance(trigger_config, JobTriggerRuleConfiguration):
logger.exception('Job type trigger rule type invalid')
raise InvalidConfiguration('Job type trigger type invalid: %s -> %s' % (result.get('name'),
trigger_rule.type))
try:
warnings.extend(trigger_config.validate_trigger_for_job(interface))
# Create a new rule when the trigger content was provided
if job_type_dict.get('trigger_rule'):
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
trigger_rule.is_active)
except (InvalidTriggerType, InvalidTriggerRule, InvalidConnection) as ex:
logger.exception('Job type trigger rule invalid')
raise InvalidConfiguration('Job type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict['trigger_rule']
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping'}
for key in job_type_dict:
if key not in base_fields:
if key in JobType.UNEDITABLE_FIELDS:
warnings.append(ValidationWarning(
'read-only', 'Job type includes read-only field: %s -> %s' % (result.get('name'), key)
))
else:
extra_fields[key] = job_type_dict[key]
# Edit or create the associated job type model
if job_type:
try:
JobType.objects.edit_job_type(job_type.id, interface, trigger_rule, remove_trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition) as ex:
logger.exception('Job type edit failed')
raise InvalidConfiguration('Unable to edit existing job type: %s -> %s' % (result.get('name'), unicode(ex)))
else:
try:
JobType.objects.create_job_type(result.get('name'), result.get('version'), interface, trigger_rule,
error_mapping, **extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition) as ex:
logger.exception('Job type create failed')
raise InvalidConfiguration('Unable to create new job type: %s -> %s' % (result.get('name'), unicode(ex)))
return warnings
def _import_job_type(job_type_dict, job_type=None):
"""Attempts to apply the given job types configuration to the system.
Note that proper model locking must be performed before calling this method.
:param job_type_dict: A dictionary of job type configuration changes to import.
:type job_type_dict: dict
:param job_type: The existing job type model to update if applicable.
:type job_type: :class:`job.models.JobType`
:returns: A list of warnings discovered during import.
:rtype: list[:class:`port.schema.ValidationWarning`]
:raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
"""
warnings = []
# Parse the JSON content into validated model fields
job_type_serializer = serializers.ConfigurationJobTypeSerializer(
job_type, data=job_type_dict)
if not job_type_serializer.is_valid():
raise InvalidConfiguration(
'Invalid job type schema: %s -> %s' %
(job_type_dict['name'], job_type_serializer.errors))
result = job_type_serializer.validated_data
# Importing system-level job types is not allowed
if result.get('category') == 'system' or (job_type and job_type.category
== 'system'):
raise InvalidConfiguration('System job types cannot be imported: %s' %
result.get('name'))
# Validate the job interface
try:
interface_dict = None
if 'interface' in result:
interface_dict = result.get('interface')
elif job_type:
interface_dict = job_type.interface
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type interface invalid: %s -> %s' %
(result.get('name'), unicode(ex)))
# Validate the error mapping
try:
error_mapping_dict = None
if 'error_mapping' in result:
error_mapping_dict = result.get('error_mapping')
elif job_type:
error_mapping_dict = job_type.error_mapping
error_mapping = ErrorInterface(error_mapping_dict)
warnings.extend(error_mapping.validate())
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' %
(result.get('name'), unicode(ex)))
# Validate the trigger rule
trigger_rule = None
if 'trigger_rule' in result and result.get('trigger_rule'):
trigger_rule = TriggerRule(**result.get('trigger_rule'))
if trigger_rule:
trigger_config = trigger_rule.get_configuration()
if not isinstance(trigger_config, JobTriggerRuleConfiguration):
logger.exception('Job type trigger rule type invalid')
raise InvalidConfiguration(
'Job type trigger type invalid: %s -> %s' %
(result.get('name'), trigger_rule.type))
try:
warnings.extend(trigger_config.validate_trigger_for_job(interface))
# Create a new rule when the trigger content was provided
if job_type_dict.get('trigger_rule'):
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule.type)
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule.configuration, trigger_rule.name,
trigger_rule.is_active)
except (InvalidTriggerType, InvalidTriggerRule,
InvalidConnection) as ex:
logger.exception('Job type trigger rule invalid')
raise InvalidConfiguration(
'Job type trigger rule invalid: %s -> %s' %
(result.get('name'), unicode(ex)))
remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict[
'trigger_rule']
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {
'name', 'version', 'interface', 'trigger_rule', 'error_mapping'
}
for key in job_type_dict:
if key not in base_fields:
if key in JobType.UNEDITABLE_FIELDS:
warnings.append(
ValidationWarning(
'read-only',
'Job type includes read-only field: %s -> %s' %
(result.get('name'), key)))
else:
extra_fields[key] = job_type_dict[key]
# Edit or create the associated job type model
if job_type:
try:
JobType.objects.edit_job_type(job_type.id, interface, trigger_rule,
remove_trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
InvalidConnection, InvalidDefinition) as ex:
logger.exception('Job type edit failed')
raise InvalidConfiguration(
'Unable to edit existing job type: %s -> %s' %
(result.get('name'), unicode(ex)))
else:
try:
JobType.objects.create_job_type(result.get('name'),
result.get('version'), interface,
trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
InvalidConnection, InvalidDefinition) as ex:
logger.exception('Job type create failed')
raise InvalidConfiguration(
'Unable to create new job type: %s -> %s' %
(result.get('name'), unicode(ex)))
return warnings
def patch(self, request, job_type_id):
"""Edits an existing job type and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param job_type_id: The ID for the job type.
:type job_type_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
# Validate the job interface
interface_dict = rest_util.parse_dict(request, 'interface', required=False)
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request, 'error_mapping', required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
error_mapping.validate()
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' % unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False)
if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or
('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)):
raise BadParameter('Trigger type and configuration are required together.')
is_active = trigger_rule_dict['is_active'] if 'is_active' in trigger_rule_dict else True
remove_trigger_rule = rest_util.has_params(request, 'trigger_rule') and not trigger_rule_dict
# Fetch the current job type model
try:
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type_id)
except JobType.DoesNotExist:
raise Http404
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for job type: %i', job_type_id)
raise BadParameter(unicode(ex))
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping'}
for key, value in request.data.iteritems():
if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
extra_fields[key] = value
try:
from recipe.configuration.definition.exceptions import InvalidDefinition
except:
logger.exception('Failed to import higher level recipe application.')
pass
try:
with transaction.atomic():
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(trigger_rule_dict['configuration'],
job_type.name, is_active)
# Update the active state separately if that is only given trigger field
if not trigger_rule and job_type.trigger_rule and 'is_active' in trigger_rule_dict:
job_type.trigger_rule.is_active = is_active
job_type.trigger_rule.save()
# Edit the job type
JobType.objects.edit_job_type(job_type_id, interface, trigger_rule, remove_trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
ValueError) as ex:
logger.exception('Unable to update job type: %i', job_type_id)
raise BadParameter(unicode(ex))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details(job_type.id)
except JobType.DoesNotExist:
raise Http404
serializer = self.get_serializer(job_type)
return Response(serializer.data)
def create(self, request):
"""Creates a new job type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
# Validate the job interface
interface_dict = rest_util.parse_dict(request, 'interface')
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request, 'error_mapping', required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
error_mapping.validate()
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' % unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False)
if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or
('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)):
raise BadParameter('Trigger type and configuration are required together.')
is_active = trigger_rule_dict['is_active'] if 'is_active' in trigger_rule_dict else True
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for new job type: %s', name)
raise BadParameter(unicode(ex))
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping'}
for key, value in request.data.iteritems():
if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
extra_fields[key] = value
try:
with transaction.atomic():
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(trigger_rule_dict['configuration'], name, is_active)
# Create the job type
job_type = JobType.objects.create_job_type(name, version, interface, trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, ValueError) as ex:
logger.exception('Unable to create new job type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details(job_type.id)
except JobType.DoesNotExist:
raise Http404
url = urlresolvers.reverse('job_type_details_view', args=[job_type.id])
serializer = JobTypeDetailsSerializer(job_type)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
class TestJobTypeManagerCreateJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.error = error_test_utils.create_error()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)
self.error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
def test_successful_no_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() successfully with no trigger rule or error mapping'''
name = 'my-job-type'
version = '1.0'
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
def test_successful_with_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() successfully with a trigger rule and error mapping'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, self.error_mapping)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
self.assertDictEqual(job_type.get_error_interface().get_dict(), self.error_mapping.get_dict())
def test_invalid_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() with an invalid trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE,
configuration=self.trigger_config.get_dict())
# Call test
self.assertRaises(InvalidConnection, JobType.objects.create_job_type, name, version, self.job_interface,
trigger_rule, self.error_mapping)
def test_successful_other_fields(self):
'''Tests calling JobTypeManager.create_job_type() successfully with additional fields'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
description=description, priority=priority)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
self.assertEqual(job_type.description, description)
self.assertEqual(job_type.priority, priority)
self.assertIsNone(job_type.archived)
self.assertIsNone(job_type.paused)
def test_successful_paused(self):
'''Tests calling JobTypeManager.create_job_type() and pausing it'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_paused = True
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
description=description, priority=priority, is_paused=is_paused)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
self.assertEqual(job_type.description, description)
self.assertEqual(job_type.priority, priority)
self.assertEqual(job_type.is_paused, is_paused)
self.assertIsNotNone(job_type.paused)
def test_uneditable_field(self):
'''Tests calling JobTypeManager.create_job_type() with an uneditable field'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
# Call test
self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, title=title,
description=description, priority=priority, is_system=is_system)
def test_invalid_error_mapping(self):
'''Tests calling JobTypeManager.create_job_type() with an invalid error mapping'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'1': 'test-invalid-error',
}
})
# Call test
self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface,
error_mapping=error_mapping, title=title, description=description, priority=priority,
is_system=is_system)