Beispiel #1
0
def _import_error(error_dict, error=None):
    """Attempts to apply the given error configuration to the system.

    Note that proper model locking must be performed before calling this method.

    :param error_dict: A dictionary of error configuration changes to import.
    :type error_dict: dict
    :param error: The existing error model to update if applicable.
    :type error: :class:`error.models.Error`
    :returns: A list of warnings discovered during import.
    :rtype: list[:class:`port.schema.ValidationWarning`]

    :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
    """
    warnings = []

    # Parse the JSON content and merge the fields into a model
    error_serializer = serializers.ConfigurationErrorSerializer(error, data=error_dict)
    if not error_serializer.is_valid():
        raise InvalidConfiguration('Invalid error schema: %s -> %s' % (error_dict.get('name'), error_serializer.errors))
    result = error_serializer.validated_data

    # Importing system-level errors is not allowed
    if result.get('category') == 'SYSTEM' or (error and error.category == 'SYSTEM'):
        raise InvalidConfiguration('System errors cannot be imported: %s' % result.get('name'))

    error_serializer.save()
    return warnings
Beispiel #2
0
def _build_error_map(error_dicts):
    """Builds a mapping of error name to existing error models.

    This method acquires a row-level lock on existing models that need to be updated.

    :param error_dicts: A list of error dictionary configuration changes.
    :type error_dicts: list[dict]
    :returns: A row-locked mapping of error names to error models.
    :rtype: dict[string, :class:`error.models.Error`]

    :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
    """
    if not error_dicts:
        return {}

    # Build a unique set of error names
    error_map = {}
    for error_dict in error_dicts:
        if 'name' not in error_dict or not error_dict['name']:
            raise InvalidConfiguration('Error import missing required field "name".')
        error_map[error_dict['name']] = None

    # Build a map of row-locked models to edit
    errors = Error.objects.select_for_update().filter(name__in=error_map.keys()).order_by('id')
    for error in errors:
        # Editing system-level errors is not allowed
        if error.category == 'SYSTEM':
            raise InvalidConfiguration('System errors cannot be edited: %s' % error.name)
        error_map[error.name] = error
    return error_map
Beispiel #3
0
def _build_job_type_map(job_type_dicts):
    """Builds a mapping of job type keys (name, version) to existing job type models.

    This method acquires a row-level lock on existing models that need to be updated.

    :param job_type_dicts: A list of job type dictionary configuration changes.
    :type job_type_dicts: list[dict]
    :returns: A row-locked mapping of job type keys (name, version) to job type models.
    :rtype: dict[tuple(string, string), :class:`job.models.JobType`]

    :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
    """
    if not job_type_dicts:
        return {}

    # Build a unique set of job type keys
    job_type_map = {}
    job_type_filters = []
    for job_type_dict in job_type_dicts:

        # Validate the required fields
        if 'name' not in job_type_dict or not job_type_dict['name']:
            raise InvalidConfiguration(
                'Job type import missing required field "name".')
        if 'version' not in job_type_dict or not job_type_dict['version']:
            raise InvalidConfiguration(
                'Job type import missing required field "version".')

        # Add the entry to the pending map
        job_type_key = (job_type_dict['name'], job_type_dict['version'])
        job_type_map[job_type_key] = None

        # Add the entry to the query set
        job_type_filter = Q(name=job_type_key[0], version=job_type_key[1])
        job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter

    # Build a map of row-locked models to edit
    job_type_query = JobType.objects.select_for_update().order_by('id')
    if job_type_filters:
        job_type_query = job_type_query.filter(job_type_filters)
    for job_type in job_type_query:
        # Editing system-level job types is not allowed
        if not job_type.category or job_type.category == 'system':
            raise InvalidConfiguration(
                'System job types cannot be edited: %s' % job_type.name)
        job_type_map[(job_type.name, job_type.version)] = job_type
    return job_type_map
Beispiel #4
0
def _build_recipe_type_map(recipe_type_dicts):
    """Builds a mapping of recipe type keys (name, version) to existing recipe type models.

    This method acquires a row-level lock on existing models that need to be updated.

    :param recipe_type_dicts: A list of recipe type dictionary configuration changes.
    :type recipe_type_dicts: list[dict]
    :returns: A row-locked mapping of recipe type keys (name, version) to recipe type models.
    :rtype: dict[tuple(string, string), :class:`recipe.models.RecipeType`]

    :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
    """
    if not recipe_type_dicts:
        return {}

    # Build a unique set of recipe type keys
    recipe_type_map = {}
    recipe_type_filters = []
    for recipe_type_dict in recipe_type_dicts:

        # Validate the required fields
        if 'name' not in recipe_type_dict or not recipe_type_dict['name']:
            raise InvalidConfiguration(
                'Recipe type import missing required field "name".')
        if 'version' not in recipe_type_dict or not recipe_type_dict['version']:
            raise InvalidConfiguration(
                'Recipe type import missing required field "version".')

        # Add the entry to the pending map
        recipe_type_key = (recipe_type_dict['name'],
                           recipe_type_dict['version'])
        recipe_type_map[recipe_type_key] = None

        # Add the entry to the query set
        recipe_type_filter = Q(name=recipe_type_key[0],
                               version=recipe_type_key[1])
        recipe_type_filters = recipe_type_filters | recipe_type_filter if recipe_type_filters else recipe_type_filter

    # Build a map of row-locked models to edit
    recipe_type_query = RecipeType.objects.select_for_update().order_by('id')
    if recipe_type_filters:
        recipe_type_query = recipe_type_query.filter(recipe_type_filters)
    for recipe_type in recipe_type_query:
        recipe_type_map[(recipe_type.name, recipe_type.version)] = recipe_type
    return recipe_type_map
Beispiel #5
0
def _import_job_type(job_type_dict, job_type=None, validating=False):
    """Attempts to apply the given job types configuration to the system.

    Note that proper model locking must be performed before calling this method.

    :param job_type_dict: A dictionary of job type configuration changes to import.
    :type job_type_dict: dict
    :param job_type: The existing job type model to update if applicable.
    :type job_type: :class:`job.models.JobType`
    :param validating: Flag to determine if running a validate or commit transaction.
    :type validating: bool
    :returns: A list of warnings discovered during import.
    :rtype: list[:class:`port.schema.ValidationWarning`]

    :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
    """
    warnings = []

    # Parse the JSON content into validated model fields
    job_type_serializer = serializers.ConfigurationJobTypeSerializer(job_type, data=job_type_dict)
    if not job_type_serializer.is_valid():
        raise InvalidConfiguration('Invalid job type schema: %s -> %s' % (job_type_dict['name'],
                                                                          job_type_serializer.errors))
    result = job_type_serializer.validated_data

    # Importing system-level job types is not allowed
    if result.get('category') == 'system' or (job_type and job_type.category == 'system'):
        raise InvalidConfiguration('System job types cannot be imported: %s' % result.get('name'))

    # Validate the job interface
    try:
        interface_dict = None
        if 'interface' in result:
            interface_dict = result.get('interface')
        elif job_type:
            interface_dict = job_type.manifest
        interface = JobInterface(interface_dict)
    except InvalidInterfaceDefinition as ex:
        raise InvalidConfiguration('Job type interface invalid: %s -> %s' % (result.get('name'), unicode(ex)))

    # Validate the job configuration
    try:
        configuration_dict = None
        secrets = None
        if 'configuration' in result:
            configuration_dict = result.get('configuration')
        elif job_type:
            configuration_dict = job_type.configuration
        if interface:
            configuration = JobConfigurationV2(configuration_dict)
            if not validating:
                secrets = configuration.get_secret_settings(interface.get_dict())
            warnings.extend(configuration.validate(interface.get_dict()))
    except InvalidJobConfiguration as ex:
        raise InvalidConfiguration('Job type configuration invalid: %s -> %s' % (result.get('name'), unicode(ex)))

    # Validate the error mapping
    try:
        error_mapping_dict = None
        if 'error_mapping' in result:
            error_mapping_dict = result.get('error_mapping')
        elif job_type:
            error_mapping_dict = job_type.error_mapping
        error_mapping = create_legacy_error_mapping(error_mapping_dict)
        error_mapping.validate_legacy()
    except InvalidInterfaceDefinition as ex:
        raise InvalidConfiguration('Job type error mapping invalid: %s -> %s' % (result.get('name'), unicode(ex)))

    # Validate the trigger rule
    trigger_rule = None
    if 'trigger_rule' in result and result.get('trigger_rule'):
        trigger_rule = TriggerRule(**result.get('trigger_rule'))
    if trigger_rule:
        trigger_config = trigger_rule.get_configuration()
        if not isinstance(trigger_config, JobTriggerRuleConfiguration):
            logger.exception('Job type trigger rule type invalid')
            raise InvalidConfiguration('Job type trigger type invalid: %s -> %s' % (result.get('name'),
                                                                                    trigger_rule.type))

        try:
            warnings.extend(trigger_config.validate_trigger_for_job(interface))

            # Create a new rule when the trigger content was provided
            if job_type_dict.get('trigger_rule'):
                rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
                trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
                                                                trigger_rule.is_active)
        except (InvalidTriggerType, InvalidTriggerRule, InvalidConnection) as ex:
            logger.exception('Job type trigger rule invalid')
            raise InvalidConfiguration('Job type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
    remove_trigger_rule = 'trigger_rule' in job_type_dict and not job_type_dict['trigger_rule']

    # Extract the fields that should be updated as keyword arguments
    extra_fields = {}
    base_fields = {'name', 'version', 'interface', 'trigger_rule', 'error_mapping', 'configuration'}
    for key in job_type_dict:
        if key not in base_fields:
            if key in JobType.UNEDITABLE_FIELDS:
                warnings.append(ValidationWarning(
                    'read-only', 'Job type includes read-only field: %s -> %s' % (result.get('name'), key)
                ))
            else:
                extra_fields[key] = job_type_dict[key]
    # Change mem_required to mem_const_required, TODO: remove once mem_required field is removed from REST API
    if 'mem_required' in extra_fields:
        extra_fields['mem_const_required'] = extra_fields['mem_required']
        del extra_fields['mem_required']

    # Edit or create the associated job type model
    if job_type:
        try:
            JobType.objects.edit_job_type_v5(job_type.id, interface=interface, trigger_rule=trigger_rule,
                                             remove_trigger_rule=remove_trigger_rule,
                                             error_mapping=error_mapping,
                                             configuration=configuration, secrets=secrets, **extra_fields)
        except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
                InvalidSecretsConfiguration) as ex:
            logger.exception('Job type edit failed')
            raise InvalidConfiguration(
                'Unable to edit existing job type: %s -> %s' % (result.get('name'), unicode(ex)))
    else:
        try:
            JobType.objects.create_job_type_v5(name=result.get('name'), version=result.get('version'),
                                               interface=interface, trigger_rule=trigger_rule,
                                               error_mapping=error_mapping, configuration=configuration,
                                               secrets=secrets, **extra_fields)
        except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, InvalidDefinition,
                InvalidSecretsConfiguration) as ex:
            logger.exception('Job type create failed')
            raise InvalidConfiguration('Unable to create new job type: %s -> %s' % (result.get('name'), unicode(ex)))
    return warnings
Beispiel #6
0
def _import_recipe_type(recipe_type_dict, recipe_type=None):
    """Attempts to apply the given recipe types configuration to the system.

    Note that proper model locking must be performed before calling this method.

    :param recipe_type_dict: A dictionary of recipe type configuration changes to import.
    :type recipe_type_dict: dict
    :param recipe_type: The existing recipe type model to update if applicable.
    :type recipe_type: :class:`recipe.models.RecipeType`
    :returns: A list of warnings discovered during import.
    :rtype: list[:class:`port.schema.ValidationWarning`]

    :raises :class:`port.schema.InvalidConfiguration`: If any part of the configuration violates the specification.
    """
    warnings = []

    # Parse the JSON content into validated model fields
    recipe_type_serializer = serializers.ConfigurationRecipeTypeSerializer(recipe_type, data=recipe_type_dict)
    if not recipe_type_serializer.is_valid():
        raise InvalidConfiguration('Invalid recipe type schema: %s -> %s' % (recipe_type_dict['name'],
                                                                             recipe_type_serializer.errors))
    result = recipe_type_serializer.validated_data

    # Validate the recipe definition
    try:
        definition_dict = None
        if 'definition' in result:
            definition_dict = result.get('definition')
        elif recipe_type:
            definition_dict = recipe_type.definition
        definition = RecipeDefinitionSunset.create(definition_dict)
        warnings.extend(definition.validate_job_interfaces())
    except (InvalidDefinition, InvalidRecipeConnection) as ex:
        raise InvalidConfiguration('Recipe type definition invalid: %s -> %s' % (result.get('name'), unicode(ex)))

    # Validate the trigger rule
    trigger_rule = None
    if 'trigger_rule' in result and result.get('trigger_rule'):
        trigger_rule = TriggerRule(**result.get('trigger_rule'))
    if trigger_rule:
        trigger_config = trigger_rule.get_configuration()
        if not isinstance(trigger_config, RecipeTriggerRuleConfiguration):
            logger.exception('Recipe type trigger rule type invalid')
            raise InvalidConfiguration('Recipe type trigger type invalid: %s -> %s' % (result.get('name'),
                                                                                       trigger_rule.type))
        try:
            warnings.extend(trigger_config.validate_trigger_for_recipe(definition))

            # Create a new rule when the trigger content was provided
            if recipe_type_dict.get('trigger_rule'):
                rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule.type)
                trigger_rule = rule_handler.create_trigger_rule(trigger_rule.configuration, trigger_rule.name,
                                                                trigger_rule.is_active)
        except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
            logger.exception('Recipe type trigger rule invalid')
            raise InvalidConfiguration('Recipe type trigger rule invalid: %s -> %s' % (result.get('name'), unicode(ex)))
    remove_trigger_rule = 'trigger_rule' in recipe_type_dict and not recipe_type_dict['trigger_rule']

    # Edit or create the associated recipe type model
    if recipe_type:
        try:
            RecipeType.objects.edit_recipe_type(recipe_type.id, result.get('title'), result.get('description'),
                                                definition, trigger_rule, remove_trigger_rule)
        except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
            logger.exception('Recipe type edit failed')
            raise InvalidConfiguration('Unable to edit recipe type: %s -> %s' % (result.get('name'), unicode(ex)))
    else:
        try:
            RecipeType.objects.create_recipe_type(result.get('name'), result.get('version'), result.get('title'),
                                                  result.get('description'), definition, trigger_rule)
        except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
            logger.exception('Recipe type create failed')
            raise InvalidConfiguration('Unable to create new recipe type: %s -> %s' % (result.get('name'), unicode(ex)))
    return warnings