def _import_job_type(job_type_dict, job_type=None, validating=False):
"""Attempts to apply the given job types configuration to the system.
Note that proper model locking must be performed before calling this method.
:param job_type_dict: A dictionary of job type configuration changes to import.
:type job_type_dict: dict
:param job_type: The existing job type model to update if applicable.
:type job_type: :class:`job.models.JobType`
:param validating: Flag to determine if running a validate or commit transaction.
:type validating: bool
:returns: A list of warnings discovered during import.
:rtype: list[:class:`port.schema.ValidationWarning`]
:raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
"""
warnings = []
# Parse the JSON content into validated model fields
job_type_serializer = serializers.ConfigurationJobTypeSerializer(job_type, data=job_type_dict)
if not job_type_serializer.is_valid():
raise InvalidConfiguration('Invalid job type schema: %s -> %s' % (job_type_dict['name'],
job_type_serializer.errors))
result = job_type_serializer.validated_data
# Importing system-level job types is not allowed
if result.get('category') == 'system' or (job_type and job_type.category == 'system'):
raise InvalidConfiguration('System job types cannot be imported: %s' % result.get('name'))
# Validate the job interface
try:
interface_dict = None
if 'interface' in result:
interface_dict = result.get('interface')
elif job_type:
interface_dict = job_type.manifest
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type interface invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the job configuration
try:
configuration_dict = None
secrets = None
if 'configuration' in result:
configuration_dict = result.get('configuration')
elif job_type:
configuration_dict = job_type.configuration
if interface:
configuration = JobConfigurationV2(configuration_dict)
if not validating:
secrets = configuration.get_secret_settings(interface.get_dict())
warnings.extend(configuration.validate(interface.get_dict()))
except InvalidJobConfiguration as ex:
raise InvalidConfiguration('Job type configuration invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the error mapping
try:
error_mapping_dict = None
if 'error_mapping' in result:
error_mapping_dict = result.get('error_mapping')
elif job_type:
error_mapping_dict = job_type.error_mapping
error_mapping = create_legacy_error_mapping(error_mapping_dict)
error_mapping.validate_legacy()
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the trigger rule
trigger_rule = None
if 'trigger_rule' in result and result.get('trigger_rule'):
trigger_rule = TriggerRule(**result.get('trigger_rule'))
if trigger_rule:
trigger_config = trigger_rule.get_configuration()
if not isinstance(trigger_config, JobTriggerRuleConfiguration):
logger.exception('Job type trigger rule type invalid')
raise InvalidConfiguration('Job type trigger type invalid: %s -> %s' % (result.get('name'),
trigger_rule.type))
try:
warnings.extend(trigger_config.validate_trigger_for_job(interface))
# Create a new rule when the trigger content was provided
if job_type_dict.get('trigger_rule'):
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
trigger_rule.is_active)
except (InvalidTriggerType, InvalidTriggerRule, InvalidConnection) as ex:
logger.exception('Job type trigger rule invalid')
raise InvalidConfiguration('Job type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict['trigger_rule']
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping', 'configuration'}
for key in job_type_dict:
if key not in base_fields:
if key in JobType.UNEDITABLE_FIELDS:
warnings.append(ValidationWarning(
'read-only', 'Job type includes read-only field: %s -> %s' % (result.get('name'), key)
))
else:
extra_fields[key] = job_type_dict[key]
# Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
if 'mem_required' in extra_fields:
extra_fields['mem_const_required'] = extra_fields['mem_required']
del extra_fields['mem_required']
# Edit or create the associated job type model
if job_type:
try:
JobType.objects.edit_job_type_v5(job_type.id, interface=interface, trigger_rule=trigger_rule,
remove_trigger_rule=remove_trigger_rule,
error_mapping=error_mapping,
configuration=configuration, secrets=secrets, **extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
InvalidSecretsConfiguration) as ex:
logger.exception('Job type edit failed')
raise InvalidConfiguration(
'Unable to edit existing job type: %s -> %s' % (result.get('name'), unicode(ex)))
else:
try:
JobType.objects.create_job_type_v5(name=result.get('name'), version=result.get('version'),
interface=interface, trigger_rule=trigger_rule,
error_mapping=error_mapping, configuration=configuration,
secrets=secrets, **extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
InvalidSecretsConfiguration) as ex:
logger.exception('Job type create failed')
raise InvalidConfiguration('Unable to create new job type: %s -> %s' % (result.get('name'), unicode(ex)))
return warnings
def _import_job_type(job_type_dict, job_type=None):
'''Attempts to apply the given job types configuration to the system.
Note that proper model locking must be performed before calling this method.
:param job_type_dict: A dictionary of job type configuration changes to import.
:type job_type_dict: dict
:param job_type: The existing job type model to update if applicable.
:type job_type: :class:`job.models.JobType`
:returns: A list of warnings discovered during import.
:rtype: list[:class:`port.schema.ValidationWarning`]
:raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
'''
warnings = []
# Parse the JSON content into validated model fields
job_type_serializer = serializers.ConfigurationJobTypeSerializer(job_type, data=job_type_dict)
if not job_type_serializer.is_valid():
raise InvalidConfiguration('Invalid job type schema: %s -> %s' % (job_type_dict['name'],
job_type_serializer.errors))
result = job_type_serializer.validated_data
# Importing system-level job types is not allowed
if result.get('category') == 'system' or (job_type and job_type.category == 'system'):
raise InvalidConfiguration('System job types cannot be imported: %s' % result.get('name'))
# Validate the job interface
try:
interface_dict = None
if 'interface' in result:
interface_dict = result.get('interface')
elif job_type:
interface_dict = job_type.interface
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type interface invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the error mapping
try:
error_mapping_dict = None
if 'error_mapping' in result:
error_mapping_dict = result.get('error_mapping')
elif job_type:
error_mapping_dict = job_type.error_mapping
error_mapping = ErrorInterface(error_mapping_dict)
warnings.extend(error_mapping.validate())
except InvalidInterfaceDefinition as ex:
raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the trigger rule
trigger_rule = None
if 'trigger_rule' in result and result.get('trigger_rule'):
trigger_rule = TriggerRule(**result.get('trigger_rule'))
if trigger_rule:
trigger_config = trigger_rule.get_configuration()
if not isinstance(trigger_config, JobTriggerRuleConfiguration):
logger.exception('Job type trigger rule type invalid')
raise InvalidConfiguration('Job type trigger type invalid: %s -> %s' % (result.get('name'),
trigger_rule.type))
try:
warnings.extend(trigger_config.validate_trigger_for_job(interface))
# Create a new rule when the trigger content was provided
if job_type_dict.get('trigger_rule'):
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
trigger_rule.is_active)
except (InvalidTriggerType, InvalidTriggerRule, InvalidConnection) as ex:
logger.exception('Job type trigger rule invalid')
raise InvalidConfiguration('Job type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict['trigger_rule']
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping'}
for key in job_type_dict:
if key not in base_fields:
if key in JobType.UNEDITABLE_FIELDS:
warnings.append(ValidationWarning(
'read-only', 'Job type includes read-only field: %s -> %s' % (result.get('name'), key)
))
else:
extra_fields[key] = job_type_dict[key]
# Edit or create the associated job type model
if job_type:
try:
JobType.objects.edit_job_type(job_type.id, interface, trigger_rule, remove_trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition) as ex:
logger.exception('Job type edit failed')
raise InvalidConfiguration('Unable to edit existing job type: %s -> %s' % (result.get('name'), unicode(ex)))
else:
try:
JobType.objects.create_job_type(result.get('name'), result.get('version'), interface, trigger_rule,
error_mapping, **extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition) as ex:
logger.exception('Job type create failed')
raise InvalidConfiguration('Unable to create new job type: %s -> %s' % (result.get('name'), unicode(ex)))
return warnings
def _import_recipe_type(recipe_type_dict, recipe_type=None):
"""Attempts to apply the given recipe types configuration to the system.
Note that proper model locking must be performed before calling this method.
:param recipe_type_dict: A dictionary of recipe type configuration changes to import.
:type recipe_type_dict: dict
:param recipe_type: The existing recipe type model to update if applicable.
:type recipe_type: :class:`recipe.models.RecipeType`
:returns: A list of warnings discovered during import.
:rtype: list[:class:`port.schema.ValidationWarning`]
:raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
"""
warnings = []
# Parse the JSON content into validated model fields
recipe_type_serializer = serializers.ConfigurationRecipeTypeSerializer(recipe_type, data=recipe_type_dict)
if not recipe_type_serializer.is_valid():
raise InvalidConfiguration('Invalid recipe type schema: %s -> %s' % (recipe_type_dict['name'],
recipe_type_serializer.errors))
result = recipe_type_serializer.validated_data
# Validate the recipe definition
try:
definition_dict = None
if 'definition' in result:
definition_dict = result.get('definition')
elif recipe_type:
definition_dict = recipe_type.definition
definition = RecipeDefinitionSunset.create(definition_dict)
warnings.extend(definition.validate_job_interfaces())
except (InvalidDefinition, InvalidRecipeConnection) as ex:
raise InvalidConfiguration('Recipe type definition invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the trigger rule
trigger_rule = None
if 'trigger_rule' in result and result.get('trigger_rule'):
trigger_rule = TriggerRule(**result.get('trigger_rule'))
if trigger_rule:
trigger_config = trigger_rule.get_configuration()
if not isinstance(trigger_config, RecipeTriggerRuleConfiguration):
logger.exception('Recipe type trigger rule type invalid')
raise InvalidConfiguration('Recipe type trigger type invalid: %s -> %s' % (result.get('name'),
trigger_rule.type))
try:
warnings.extend(trigger_config.validate_trigger_for_recipe(definition))
# Create a new rule when the trigger content was provided
if recipe_type_dict.get('trigger_rule'):
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
trigger_rule.is_active)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Recipe type trigger rule invalid')
raise InvalidConfiguration('Recipe type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
remove_trigger_rule = 'trigger_rule' in recipe_type_dict and not recipe_type_dict['trigger_rule']
# Edit or create the associated recipe type model
if recipe_type:
try:
RecipeType.objects.edit_recipe_type(recipe_type.id, result.get('title'), result.get('description'),
definition, trigger_rule, remove_trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Recipe type edit failed')
raise InvalidConfiguration('Unable to edit recipe type: %s -> %s' % (result.get('name'), unicode(ex)))
else:
try:
RecipeType.objects.create_recipe_type(result.get('name'), result.get('version'), result.get('title'),
result.get('description'), definition, trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Recipe type create failed')
raise InvalidConfiguration('Unable to create new recipe type: %s -> %s' % (result.get('name'), unicode(ex)))
return warnings
def _import_recipe_type(recipe_type_dict, recipe_type=None):
'''Attempts to apply the given recipe types configuration to the system.
Note that proper model locking must be performed before calling this method.
:param recipe_type_dict: A dictionary of recipe type configuration changes to import.
:type recipe_type_dict: dict
:param recipe_type: The existing recipe type model to update if applicable.
:type recipe_type: :class:`recipe.models.RecipeType`
:returns: A list of warnings discovered during import.
:rtype: list[:class:`port.schema.ValidationWarning`]
:raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
'''
warnings = []
# Parse the JSON content into validated model fields
recipe_type_serializer = serializers.ConfigurationRecipeTypeSerializer(recipe_type, data=recipe_type_dict)
if not recipe_type_serializer.is_valid():
raise InvalidConfiguration('Invalid recipe type schema: %s -> %s' % (recipe_type_dict['name'],
recipe_type_serializer.errors))
result = recipe_type_serializer.validated_data
# Validate the recipe definition
try:
definition_dict = None
if 'definition' in result:
definition_dict = result.get('definition')
elif recipe_type:
definition_dict = recipe_type.definition
definition = RecipeDefinition(definition_dict)
warnings.extend(definition.validate_job_interfaces())
except (InvalidDefinition, InvalidRecipeConnection) as ex:
raise InvalidConfiguration('Recipe type definition invalid: %s -> %s' % (result.get('name'), unicode(ex)))
# Validate the trigger rule
trigger_rule = None
if 'trigger_rule' in result and result.get('trigger_rule'):
trigger_rule = TriggerRule(**result.get('trigger_rule'))
if trigger_rule:
trigger_config = trigger_rule.get_configuration()
if not isinstance(trigger_config, RecipeTriggerRuleConfiguration):
logger.exception('Recipe type trigger rule type invalid')
raise InvalidConfiguration('Recipe type trigger type invalid: %s -> %s' % (result.get('name'),
trigger_rule.type))
try:
warnings.extend(trigger_config.validate_trigger_for_recipe(definition))
# Create a new rule when the trigger content was provided
if recipe_type_dict.get('trigger_rule'):
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
trigger_rule.is_active)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Recipe type trigger rule invalid')
raise InvalidConfiguration('Recipe type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
remove_trigger_rule = 'trigger_rule' in recipe_type_dict and not recipe_type_dict['trigger_rule']
# Edit or create the associated recipe type model
if recipe_type:
try:
RecipeType.objects.edit_recipe_type(recipe_type.id, result.get('title'), result.get('description'),
definition, trigger_rule, remove_trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Recipe type edit failed')
raise InvalidConfiguration('Unable to edit recipe type: %s -> %s' % (result.get('name'), unicode(ex)))
else:
try:
RecipeType.objects.create_recipe_type(result.get('name'), result.get('version'), result.get('title'),
result.get('description'), definition, trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Recipe type create failed')
raise InvalidConfiguration('Unable to create new recipe type: %s -> %s' % (result.get('name'), unicode(ex)))
return warnings