def __init__(self, definition=None):
"""Creates a configuration interface from the given definition.
If the definition is invalid, a :class:`job.configuration.interface.exceptions.InvalidInterfaceDefinition`
exception will be thrown.
:param definition: The interface definition
:type definition: dict
"""
if definition is None:
definition = {}
self.definition = definition
self._default_setting_names = set()
try:
validate(definition, JOB_CONFIG_SCHEMA)
except ValidationError as validation_error:
raise InvalidJobConfiguration('INVALID_CONFIGURATION', validation_error)
self._populate_default_values()
self._validate_default_settings()
if self.definition['version'] != '1.0':
msg = '%s is an unsupported version number'
raise InvalidJobConfiguration('INVALID_VERSION', msg % self.definition['version'])
def _validate_default_settings(self):
"""Ensures that no settings have duplicate names or blank values
:raises :class:`job.configuration.interface.exceptions.InvalidInterface`: If there is a duplicate name or blank
value/name.
"""
for setting_name, setting_value in self.definition[
'default_settings'].iteritems():
if setting_name in self._default_setting_names:
raise InvalidJobConfiguration(
'Duplicate setting name %s in default_settings' %
setting_name)
self._default_setting_names.add(setting_name)
if not setting_name:
raise InvalidJobConfiguration(
'Blank setting name (value = %s) in default_settings' %
setting_value)
if not setting_value:
raise InvalidJobConfiguration(
'Blank setting value (name = %s) in default_settings' %
setting_name)
if not isinstance(setting_value, basestring):
raise InvalidJobConfiguration(
'Setting value (name = %s) is not a string' % setting_name)
def _validate_output_workspaces(self, manifest):
"""Validates the workspace configuration against the given Seed manifest. This will perform database queries.
:param manifest: The Seed manifest
:type manifest: :class:`job.seed.manifest.SeedManifest`
:returns: A list of warnings discovered during validation
:rtype: list
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid
"""
warnings = []
seed_outputs = {
output_dict['name']
for output_dict in manifest.get_output_files()
}
seed_outputs.update({
output_dict['name']
for output_dict in manifest.get_output_files()
})
workspace_names = set(self.output_workspaces.values())
if self.default_output_workspace:
workspace_names.add(self.default_output_workspace)
workspace_models = {
w.name: w
for w in Workspace.objects.get_workspaces(names=workspace_names)
}
for name in workspace_names:
if name not in workspace_models:
raise InvalidJobConfiguration(
'INVALID_WORKSPACE', 'No workspace named \'%s\'' % name)
if not workspace_models[name].is_active:
warnings.append(
ValidationWarning('DEPRECATED_WORKSPACE',
'Workspace \'%s\' is deprecated' % name))
# TODO: add RO/RW mode to workspaces and add warning if using a RO workspace for output
missing_workspace_names = []
if not self.default_output_workspace:
for name in seed_outputs:
if name not in self.output_workspaces:
missing_workspace_names.append(name)
if missing_workspace_names:
msg = 'Missing workspace for outputs \'%s\''
raise InvalidJobConfiguration('MISSING_WORKSPACE',
msg % missing_workspace_names)
return warnings
def __init__(self, configuration=None, do_validate=True):
"""Creates a job configuration from the given dict
:param configuration: The configuration dict
:type configuration: dict
:param do_validate: Whether to perform validation on the JSON schema
:type do_validate: bool
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the given configuration is invalid
"""
if configuration is None:
configuration = {}
self._configuration = configuration
if 'version' not in self._configuration:
self._configuration['version'] = SCHEMA_VERSION
if self._configuration['version'] != SCHEMA_VERSION:
self._convert_configuration()
try:
if do_validate:
validate(configuration, JOB_CONFIG_SCHEMA)
except ValidationError as validation_error:
raise InvalidJobConfiguration('INVALID_CONFIGURATION', validation_error)
self._populate_default_values()
self._validate_mounts()
self._validate_settings()
def __init__(self, configuration=None):
"""Creates a job configuration from the given dict
:param configuration: The configuration dict
:type configuration: dict
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the given configuration is invalid
"""
if configuration is None:
configuration = {}
self._configuration = configuration
if 'version' not in self._configuration:
self._configuration['version'] = SCHEMA_VERSION
if self._configuration['version'] != SCHEMA_VERSION:
self._convert_configuration()
try:
validate(configuration, JOB_CONFIG_SCHEMA)
except ValidationError as validation_error:
raise InvalidJobConfiguration(validation_error)
self._populate_default_values()
self._validate_mounts()
self._validate_settings()
def __init__(self, config=None, do_validate=False):
"""Creates a v6 job configuration JSON object from the given dictionary
:param config: The job configuration JSON dict
:type config: dict
:param do_validate: Whether to perform validation on the JSON schema
:type do_validate: bool
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the given configuration is invalid
"""
if not config:
config = {}
self._config = config
if 'version' not in self._config:
self._config['version'] = SCHEMA_VERSION
if self._config['version'] != SCHEMA_VERSION:
self._convert_from_v2(do_validate)
self._populate_default_values()
try:
if do_validate:
validate(self._config, JOB_CONFIG_SCHEMA)
except ValidationError as ex:
raise InvalidJobConfiguration(
'INVALID_CONFIGURATION',
'Invalid configuration: %s' % unicode(ex))
def _validate_settings(self):
"""Ensures that the settings are valid
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the settings are invalid
"""
for setting_name, setting_value in self._configuration['settings'].iteritems():
if not setting_value:
raise InvalidJobConfiguration('INVALID_CONFIGURATION', 'Setting %s has blank value' % setting_name)
def validate(self):
"""See :meth:`job.configuration.mount.MountConfig.validate`
"""
warnings = super(HostMountConfig, self).validate()
if not os.path.isabs(self.host_path):
msg = 'Host mount %s must use an absolute host path'
raise InvalidJobConfiguration('HOST_ABSOLUTE_PATH', msg % self.name)
return warnings
def add_setting(self, setting_name, setting_value):
"""Adds the given setting value
:param setting_name: The setting name
:type setting_name: string
:param setting_value: The setting value
:type setting_value: string
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the setting is a duplicate or invalid
"""
if setting_name in self.settings:
raise InvalidJobConfiguration(
'DUPLICATE_SETTING', 'Duplicate setting \'%s\'' % setting_name)
if setting_value is None:
msg = 'The value for setting \'%s\' must be a non-empty string'
raise InvalidJobConfiguration('INVALID_SETTING',
msg % setting_name)
self.settings[setting_name] = setting_value
def __init__(self, config=None, existing=None, do_validate=False):
"""Creates a v6 job configuration JSON object from the given dictionary
:param config: The job configuration JSON dict
:type config: dict
:param existing: Existing JobConfiguration to use for default values for unspecified fields
:type existing: JobConfigurationV6
:param do_validate: Whether to perform validation on the JSON schema
:type do_validate: bool
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the given configuration is invalid
"""
if not config:
config = {}
self._config = config
self._existing_config = None
if existing:
self._existing_config = existing._config
if 'version' not in self._config:
self._config['version'] = SCHEMA_VERSION
if self._config['version'] == '2.0':
# v2 and v6 are identical
self._config['version'] = SCHEMA_VERSION
if self._config['version'] not in SCHEMA_VERSIONS:
raise InvalidJobConfiguration(
'INVALID_VERSION', 'Invalid configuration version: %s' %
unicode(self._config['version']))
self._populate_default_values()
try:
if do_validate:
validate(self._config, JOB_CONFIG_SCHEMA)
except ValidationError as ex:
raise InvalidJobConfiguration(
'INVALID_CONFIGURATION',
'Invalid configuration: %s' % unicode(ex))
def _validate_mounts(self):
"""Ensures that the mounts are valid
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the mounts are invalid
"""
for name, mount in self._configuration['mounts'].iteritems():
if mount['type'] == 'host':
if 'host_path' not in mount:
raise InvalidJobConfiguration('INVALID_CONFIGURATION', 'Host mount %s requires host_path' % name)
if not os.path.isabs(mount['host_path']):
msg = 'Host mount %s must use an absolute host_path'
raise InvalidJobConfiguration('INVALID_CONFIGURATION', msg % name)
if 'driver' in mount:
msg = 'Host mount %s does not support driver'
raise InvalidJobConfiguration('INVALID_CONFIGURATION', msg % name)
if 'driver_opts' in mount:
msg = 'Host mount %s does not support driver_opts'
raise InvalidJobConfiguration('INVALID_CONFIGURATION', msg % name)
elif mount['type'] == 'volume':
if 'driver' not in mount:
raise InvalidJobConfiguration('INVALID_CONFIGURATION', 'Volume mount %s requires driver' % name)
if 'host_path' in mount:
msg = 'Volume mount %s does not support host_path'
raise InvalidJobConfiguration('INVALID_CONFIGURATION', msg % name)
def add_mount(self, mount_config):
"""Adds the given mount configuration
:param mount_config: The mount configuration to add
:type mount_config: :class:`job.configuration.mount.MountConfig`
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the mount is a duplicate
"""
if mount_config.name in self.mounts:
raise InvalidJobConfiguration(
'DUPLICATE_MOUNT',
'Duplicate mount \'%s\'' % mount_config.name)
self.mounts[mount_config.name] = mount_config
def add_output_workspace(self, output, workspace):
"""Adds the given output_workspace
:param output: The output name
:type output: string
:param workspace: The workspace name
:type workspace: string
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the output is a duplicate
"""
if output in self.output_workspaces:
raise InvalidJobConfiguration(
'DUPLICATE_WORKSPACE',
'Duplicate output workspace \'%s\'' % output)
self.output_workspaces[output] = workspace
def validate(self, manifest):
"""Validates the configuration against the given Seed manifest. This will perform database queries.
:param manifest: The Seed manifest
:type manifest: :class:`job.seed.manifest.SeedManifest`
:returns: A list of warnings discovered during validation
:rtype: list
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid
"""
if self.priority < 1:
raise InvalidJobConfiguration(
'INVALID_PRIORITY', 'Priority must be a positive integer')
warnings = self._validate_mounts(manifest)
warnings.extend(self._validate_output_workspaces(manifest))
warnings.extend(self._validate_settings(manifest))
return warnings