def _validate_job_dependencies(self): """Validates that every job dependency is listed in jobs and that there are no cyclic dependencies :raises InvalidDefinition: If there is an undefined job or a cyclic dependency """ # Make sure all dependencies are defined for job_dict in self._definition['jobs']: job_name = job_dict['name'] for dependency_dict in job_dict['dependencies']: dependency_name = dependency_dict['name'] if dependency_name not in self._jobs_by_name: msg = 'Invalid recipe definition: Job %s has undefined dependency %s' % (job_name, dependency_name) raise InvalidDefinition(msg) # Ensure no cyclic dependencies for job_dict in self._definition['jobs']: job_name = job_dict['name'] dependencies_to_check = set() dependencies_to_check.add(job_name) while dependencies_to_check: next_layer = set() for dependency in dependencies_to_check: job_dict = self._jobs_by_name[dependency] for dependency_dict in job_dict['dependencies']: dependency_name = dependency_dict['name'] if dependency_name == job_name: msg = 'Invalid recipe definition: Job %s has a circular dependency' % job_name raise InvalidDefinition(msg) next_layer.add(dependency_name) dependencies_to_check = next_layer
def validate_job_interfaces(self): """Validates the interfaces of the recipe jobs in the definition to ensure that all of the input and output connections are valid :returns: A list of warnings discovered during validation. :rtype: list[:class:`job.configuration.data.job_data.ValidationWarning`] :raises :class:`recipe.configuration.definition.exceptions.InvalidDefinition`: If there are any invalid job connections in the definition """ # Query for job types job_types_by_name = self.get_job_type_map() # Job name in recipe -> job type model for job_name, job_data in self._jobs_by_name.iteritems(): if job_name not in job_types_by_name: if 'job_type' in job_data: job_type = job_data['job_type'] if 'name' in job_type and 'version' in job_type: raise InvalidDefinition('Unknown job type: (%s, %s)' % (job_type['name'], job_type['version'])) else: raise InvalidDefinition('Missing job type name or version: %s' % job_name) else: raise InvalidDefinition('Missing job type declaration: %s' % job_name) warnings = [] for job_name in self._jobs_by_name: job_dict = self._jobs_by_name[job_name] warnings.extend(self._validate_job_interface(job_dict, job_types_by_name)) return warnings
def __init__(self, definition): """Creates a recipe definition object from the given dictionary. The general format is checked for correctness, but the actual job details are not checked for correctness. :param definition: The recipe definition :type definition: dict :raises InvalidDefinition: If the given definition is invalid """ self._definition = definition self._input_files_by_name = { } # Name -> `job.seed.types.SeedInputFiles` self._input_json_by_name = {} # Name -> `job.seed.types.SeedInputJson` self._jobs_by_name = {} # Name -> job dict self._property_validation_dict = {} # Property Input name -> required self._input_file_validation_dict = { } # File Input name -> (required, multiple, file description) try: validate(definition, RECIPE_DEFINITION_SCHEMA) except ValidationError as ex: raise InvalidDefinition('Invalid recipe definition: %s' % unicode(ex)) self._populate_default_values() if not self._definition['version'] == DEFAULT_VERSION: raise InvalidDefinition('%s is an unsupported version number' % self._definition['version']) for input_file in self._get_input_files(): name = input_file['name'] if name in self._input_files_by_name: raise InvalidDefinition( 'Invalid recipe definition: %s is a duplicate input data name' % name) self._input_files_by_name[name] = SeedInputFiles(input_file) for input_json in self._get_input_json(): name = input_json['name'] if name in self._input_json_by_name or name in self._input_files_by_name: raise InvalidDefinition( 'Invalid recipe definition: %s is a duplicate input data name' % name) self._input_json_by_name[name] = SeedInputJson(input_json) for job_dict in self._definition['jobs']: name = job_dict['name'] if name in self._jobs_by_name: raise InvalidDefinition( 'Invalid recipe definition: %s is a duplicate job name' % name) self._jobs_by_name[name] = job_dict self._create_validation_dicts() self._validate_job_dependencies() self._validate_no_dup_job_inputs() self._validate_recipe_inputs()
def _validate_recipe_inputs(self): """Validates that the recipe inputs used when listing the jobs are defined in the input data section :raises InvalidDefinition: If there is an undefined recipe input """ for job_dict in self._definition['jobs']: job_name = job_dict['name'] for recipe_dict in job_dict['recipe_inputs']: recipe_input = recipe_dict['recipe_input'] if recipe_input not in self._inputs_by_name: msg = 'Invalid recipe definition: Job %s has undefined recipe input %s' % (job_name, recipe_input) raise InvalidDefinition(msg)
def _validate_no_dup_job_inputs(self): """Validates that there are no duplicate inputs for any job :raises InvalidDefinition: If there is a duplicate input """ for job_dict in self._definition['jobs']: job_name = job_dict['name'] input_names = set() for recipe_dict in job_dict['recipe_inputs']: name = recipe_dict['job_input'] if name in input_names: msg = 'Invalid recipe definition: Job %s has duplicate input %s' % (job_name, name) raise InvalidDefinition(msg) input_names.add(name) for dependency_dict in job_dict['dependencies']: for conn_dict in dependency_dict['connections']: name = conn_dict['input'] if name in input_names: msg = 'Invalid recipe definition: Job %s has duplicate input %s' % (job_name, name) raise InvalidDefinition(msg) input_names.add(name)
def _validate_job_interface(self, job_dict, job_types_by_name): """Validates the input connections for the given job in the recipe definition :param job_dict: The job dictionary :type job_dict: dict :param job_types_by_name: Dict mapping all job names in the recipe to their job type models :type job_types_by_name: dict :returns: A list of warnings discovered during validation. :rtype: list[:class:`job.configuration.data.job_data.ValidationWarning`] :raises :class:`recipe.configuration.definition.exceptions.InvalidDefinition`: If there are any invalid job connections in the definition """ job_type = job_types_by_name[job_dict['name']] # Job connection will represent data to be passed to the job to validate job_conn = JobConnectionSunset.create(job_type.get_job_interface()) # Assume a workspace is provided, this will be verified when validating the recipe data job_conn.add_workspace() # Populate connection with data that will come from recipe inputs self._add_recipe_inputs_to_conn(job_conn, job_dict['recipe_inputs']) # Populate connection with data that will come from job dependencies warnings = [] for dependency_dict in job_dict['dependencies']: dependency_name = dependency_dict['name'] job_type = job_types_by_name[dependency_name] for conn_dict in dependency_dict['connections']: conn_input = conn_dict['input'] job_output = conn_dict['output'] job_type.get_job_interface().add_output_to_connection( job_output, job_conn, conn_input) job_type = job_types_by_name[job_dict['name']] try: warnings.extend( job_type.get_job_interface().validate_connection(job_conn)) except InvalidConnection as ex: raise InvalidDefinition(unicode(ex)) return warnings