def _validate_job_dependencies(self):
"""Validates that every job dependency is listed in jobs and that there are no cyclic dependencies
:raises InvalidDefinition: If there is an undefined job or a cyclic dependency
"""
# Make sure all dependencies are defined
for job_dict in self._definition['jobs']:
job_name = job_dict['name']
for dependency_dict in job_dict['dependencies']:
dependency_name = dependency_dict['name']
if dependency_name not in self._jobs_by_name:
msg = 'Invalid recipe definition: Job %s has undefined dependency %s' % (job_name, dependency_name)
raise InvalidDefinition(msg)
# Ensure no cyclic dependencies
for job_dict in self._definition['jobs']:
job_name = job_dict['name']
dependencies_to_check = set()
dependencies_to_check.add(job_name)
while dependencies_to_check:
next_layer = set()
for dependency in dependencies_to_check:
job_dict = self._jobs_by_name[dependency]
for dependency_dict in job_dict['dependencies']:
dependency_name = dependency_dict['name']
if dependency_name == job_name:
msg = 'Invalid recipe definition: Job %s has a circular dependency' % job_name
raise InvalidDefinition(msg)
next_layer.add(dependency_name)
dependencies_to_check = next_layer
def validate_job_interfaces(self):
"""Validates the interfaces of the recipe jobs in the definition to ensure that all of the input and output
connections are valid
:returns: A list of warnings discovered during validation.
:rtype: list[:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`recipe.configuration.definition.exceptions.InvalidDefinition`:
If there are any invalid job connections in the definition
"""
# Query for job types
job_types_by_name = self.get_job_type_map() # Job name in recipe -> job type model
for job_name, job_data in self._jobs_by_name.iteritems():
if job_name not in job_types_by_name:
if 'job_type' in job_data:
job_type = job_data['job_type']
if 'name' in job_type and 'version' in job_type:
raise InvalidDefinition('Unknown job type: (%s, %s)' % (job_type['name'], job_type['version']))
else:
raise InvalidDefinition('Missing job type name or version: %s' % job_name)
else:
raise InvalidDefinition('Missing job type declaration: %s' % job_name)
warnings = []
for job_name in self._jobs_by_name:
job_dict = self._jobs_by_name[job_name]
warnings.extend(self._validate_job_interface(job_dict, job_types_by_name))
return warnings
def __init__(self, definition):
"""Creates a recipe definition object from the given dictionary. The general format is checked for correctness,
but the actual job details are not checked for correctness.
:param definition: The recipe definition
:type definition: dict
:raises InvalidDefinition: If the given definition is invalid
"""
self._definition = definition
self._input_files_by_name = {
} # Name -> `job.seed.types.SeedInputFiles`
self._input_json_by_name = {} # Name -> `job.seed.types.SeedInputJson`
self._jobs_by_name = {} # Name -> job dict
self._property_validation_dict = {} # Property Input name -> required
self._input_file_validation_dict = {
} # File Input name -> (required, multiple, file description)
try:
validate(definition, RECIPE_DEFINITION_SCHEMA)
except ValidationError as ex:
raise InvalidDefinition('Invalid recipe definition: %s' %
unicode(ex))
self._populate_default_values()
if not self._definition['version'] == DEFAULT_VERSION:
raise InvalidDefinition('%s is an unsupported version number' %
self._definition['version'])
for input_file in self._get_input_files():
name = input_file['name']
if name in self._input_files_by_name:
raise InvalidDefinition(
'Invalid recipe definition: %s is a duplicate input data name'
% name)
self._input_files_by_name[name] = SeedInputFiles(input_file)
for input_json in self._get_input_json():
name = input_json['name']
if name in self._input_json_by_name or name in self._input_files_by_name:
raise InvalidDefinition(
'Invalid recipe definition: %s is a duplicate input data name'
% name)
self._input_json_by_name[name] = SeedInputJson(input_json)
for job_dict in self._definition['jobs']:
name = job_dict['name']
if name in self._jobs_by_name:
raise InvalidDefinition(
'Invalid recipe definition: %s is a duplicate job name' %
name)
self._jobs_by_name[name] = job_dict
self._create_validation_dicts()
self._validate_job_dependencies()
self._validate_no_dup_job_inputs()
self._validate_recipe_inputs()
def _validate_recipe_inputs(self):
"""Validates that the recipe inputs used when listing the jobs are defined in the input data section
:raises InvalidDefinition: If there is an undefined recipe input
"""
for job_dict in self._definition['jobs']:
job_name = job_dict['name']
for recipe_dict in job_dict['recipe_inputs']:
recipe_input = recipe_dict['recipe_input']
if recipe_input not in self._inputs_by_name:
msg = 'Invalid recipe definition: Job %s has undefined recipe input %s' % (job_name, recipe_input)
raise InvalidDefinition(msg)
def _validate_no_dup_job_inputs(self):
"""Validates that there are no duplicate inputs for any job
:raises InvalidDefinition: If there is a duplicate input
"""
for job_dict in self._definition['jobs']:
job_name = job_dict['name']
input_names = set()
for recipe_dict in job_dict['recipe_inputs']:
name = recipe_dict['job_input']
if name in input_names:
msg = 'Invalid recipe definition: Job %s has duplicate input %s' % (job_name, name)
raise InvalidDefinition(msg)
input_names.add(name)
for dependency_dict in job_dict['dependencies']:
for conn_dict in dependency_dict['connections']:
name = conn_dict['input']
if name in input_names:
msg = 'Invalid recipe definition: Job %s has duplicate input %s' % (job_name, name)
raise InvalidDefinition(msg)
input_names.add(name)
def _validate_job_interface(self, job_dict, job_types_by_name):
"""Validates the input connections for the given job in the recipe definition
:param job_dict: The job dictionary
:type job_dict: dict
:param job_types_by_name: Dict mapping all job names in the recipe to their job type models
:type job_types_by_name: dict
:returns: A list of warnings discovered during validation.
:rtype: list[:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`recipe.configuration.definition.exceptions.InvalidDefinition`:
If there are any invalid job connections in the definition
"""
job_type = job_types_by_name[job_dict['name']]
# Job connection will represent data to be passed to the job to validate
job_conn = JobConnectionSunset.create(job_type.get_job_interface())
# Assume a workspace is provided, this will be verified when validating the recipe data
job_conn.add_workspace()
# Populate connection with data that will come from recipe inputs
self._add_recipe_inputs_to_conn(job_conn, job_dict['recipe_inputs'])
# Populate connection with data that will come from job dependencies
warnings = []
for dependency_dict in job_dict['dependencies']:
dependency_name = dependency_dict['name']
job_type = job_types_by_name[dependency_name]
for conn_dict in dependency_dict['connections']:
conn_input = conn_dict['input']
job_output = conn_dict['output']
job_type.get_job_interface().add_output_to_connection(
job_output, job_conn, conn_input)
job_type = job_types_by_name[job_dict['name']]
try:
warnings.extend(
job_type.get_job_interface().validate_connection(job_conn))
except InvalidConnection as ex:
raise InvalidDefinition(unicode(ex))
return warnings
