def _validate_settings(self, manifest):
"""Validates the setting configuration against the given Seed manifest
:param manifest: The Seed manifest
:type manifest: :class:`job.seed.manifest.SeedManifest`
:returns: A list of warnings discovered during validation
:rtype: list
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid
"""
warnings = []
seed_settings = {
setting_dict['name']
for setting_dict in manifest.get_settings()
}
for name in self.settings:
if name not in seed_settings:
warnings.append(
ValidationWarning('UNKNOWN_SETTING',
'Unknown setting \'%s\' ignored' % name))
for name in seed_settings:
if name not in self.settings:
warnings.append(
ValidationWarning(
'MISSING_SETTING',
'Missing configuration for setting \'%s\'' % name))
return warnings
def _validate_mounts(self, manifest):
"""Validates the mount configuration against the given Seed manifest
:param manifest: The Seed manifest
:type manifest: :class:`job.seed.manifest.SeedManifest`
:returns: A list of warnings discovered during validation
:rtype: list
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid
"""
warnings = []
seed_mounts = {
mount_dict['name']
for mount_dict in manifest.get_mounts()
}
for mount_config in self.mounts.values():
warnings.extend(mount_config.validate())
if mount_config.name not in seed_mounts:
warnings.append(
ValidationWarning(
'UNKNOWN_MOUNT',
'Unknown mount \'%s\' ignored' % mount_config.name))
for name in seed_mounts:
if name not in self.mounts:
warnings.append(
ValidationWarning(
'MISSING_MOUNT',
'Missing configuration for mount \'%s\'' % name))
return warnings
def validate_configuration(self, configuration):
"""See :meth:`ingest.strike.monitors.monitor.Monitor.validate_configuration`
"""
warnings = []
if 'sqs_name' not in configuration:
raise InvalidMonitorConfiguration('sqs_name is required for s3 monitor')
if not isinstance(configuration['sqs_name'], basestring):
raise InvalidMonitorConfiguration('sqs_name must be a string')
if not configuration['sqs_name']:
raise InvalidMonitorConfiguration('sqs_name must be a non-empty string')
# If credentials exist, validate them.
credentials = AWSClient.instantiate_credentials_from_config(configuration)
region_name = configuration.get('region_name')
# Check whether the queue can actually be accessed
with SQSClient(credentials, region_name) as client:
try:
client.get_queue_by_name(configuration['sqs_name'])
except ClientError:
warnings.append(ValidationWarning('sqs_access',
'Unable to access SQS. Check the name, region and credentials.'))
return warnings
def _validate_output_workspaces(self, manifest):
"""Validates the workspace configuration against the given Seed manifest. This will perform database queries.
:param manifest: The Seed manifest
:type manifest: :class:`job.seed.manifest.SeedManifest`
:returns: A list of warnings discovered during validation
:rtype: list
:raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid
"""
warnings = []
seed_outputs = {
output_dict['name']
for output_dict in manifest.get_outputs()['files']
}
seed_outputs.update({
output_dict['name']
for output_dict in manifest.get_outputs()['files']
})
workspace_names = set(self.output_workspaces.values())
if self.default_output_workspace:
workspace_names.add(self.default_output_workspace)
workspace_models = {
w.name: w
for w in Workspace.objects.get_workspaces(names=workspace_names)
}
for name in workspace_names:
if name not in workspace_models:
raise InvalidJobConfiguration(
'INVALID_WORKSPACE', 'No workspace named \'%s\'' % name)
if not workspace_models[name].is_active:
warnings.append(
ValidationWarning('DEPRECATED_WORKSPACE',
'Workspace \'%s\' is deprecated' % name))
# TODO: add RO/RW mode to workspaces and add warning if using a RO workspace for output
if not self.default_output_workspace:
for name in seed_outputs:
if name not in self.output_workspaces:
msg = 'Missing workspace for output \'%s\''
warnings.append(
ValidationWarning('MISSING_WORKSPACE', msg % name))
return warnings
def validate_configuration(self, configuration):
"""See :meth:`ingest.strike.monitors.monitor.Monitor.validate_configuration`
"""
warnings = []
if 'transfer_suffix' in configuration:
if not isinstance(configuration['transfer_suffix'], basestring):
raise InvalidMonitorConfiguration('transfer_suffix must be a string')
if not configuration['transfer_suffix']:
raise InvalidMonitorConfiguration('transfer_suffix must be a non-empty string')
if 'transfer_suffix' not in configuration:
warnings.append(ValidationWarning('missing_transfer_suffix',
'transfer_suffix is not specified. Using default value of "_tmp"'))
return warnings
def validate_configuration(self, config):
"""See :meth:`storage.brokers.broker.Broker.validate_configuration`"""
warnings = []
if 'bucket_name' not in config or not config['bucket_name']:
raise InvalidBrokerConfiguration('INVALID_BROKER', 'S3 broker requires "bucket_name" to be populated')
region_name = config.get('region_name')
credentials = AWSClient.instantiate_credentials_from_config(config)
# Check whether the bucket can actually be accessed
with S3Client(credentials, region_name) as client:
try:
client.get_bucket(config['bucket_name'])
except (ClientError, NoCredentialsError):
warnings.append(ValidationWarning('bucket_access',
'Unable to access bucket. Check the bucket name and credentials.'))
return warnings
def validate_connection(self, connecting_parameter):
"""See :meth:`data.interface.parameter.Parameter.validate_connection`
"""
warnings = super(FileParameter, self).validate_connection(connecting_parameter)
if not self.multiple and connecting_parameter.multiple:
msg = 'Parameter \'%s\' cannot accept multiple files' % self.name
raise InvalidInterfaceConnection('NO_MULTIPLE_FILES', msg)
mismatched_media_types = []
for media_type in connecting_parameter.media_types:
if media_type not in self.media_types:
mismatched_media_types.append(media_type)
if mismatched_media_types:
msg = 'Parameter \'%s\' might not accept [%s]' % (self.name, ', '.join(mismatched_media_types))
warnings.append(ValidationWarning('MISMATCHED_MEDIA_TYPES', msg))
return warnings
def validate(self, interface):
"""Validates this data filter against the given interface
:param interface: The interface describing the data that will be passed to the filter
:type interface: :class:`data.interface.interface.Interface`
:returns: A list of warnings discovered during validation
:rtype: :func:`list`
:raises :class:`data.filter.exceptions.InvalidDataFilter`: If the data filter is invalid
"""
warnings = []
unmatched = interface.parameters.keys()
for f in self.filter_list:
name = f['name']
filter_type = f['type']
if name in interface.parameters:
if name in unmatched:
unmatched.remove(name)
if interface.parameters[
name].param_type == 'file' and filter_type not in FILE_TYPES:
raise InvalidDataFilter(
'MISMATCHED_TYPE',
'Interface parameter is a file type and requires a file type filter.'
)
if interface.parameters[
name].param_type == 'json' and filter_type in FILE_TYPES:
raise InvalidDataFilter(
'MISMATCHED_TYPE',
'Interface parameter is a json type and will not work with a file type filter.'
)
if interface.parameters[name].param_type == 'json':
if interface.parameters[
name].json_type in STRING_TYPES and filter_type not in STRING_TYPES:
raise InvalidDataFilter(
'MISMATCHED_TYPE',
'Interface parameter is a string and filter is not a string type filter'
)
if interface.parameters[
name].json_type in NUMBER_TYPES and filter_type not in NUMBER_TYPES:
raise InvalidDataFilter(
'MISMATCHED_TYPE',
'Interface parameter is a number and filter is not a number type filter'
)
if interface.parameters[
name].json_type in BOOL_TYPES and filter_type not in BOOL_TYPES:
raise InvalidDataFilter(
'MISMATCHED_TYPE',
'Interface parameter is a number and filter is not a number type filter'
)
json_type = interface.parameters[name].json_type
if json_type not in BOOL_TYPES and json_type not in STRING_TYPES and json_type not in NUMBER_TYPES:
raise InvalidDataFilter(
'MISMATCHED_TYPE',
'Interface parameter type is not supported by data filters'
)
else:
warnings.append(
ValidationWarning(
'UNMATCHED_FILTER',
'Filter with name \'%s\' does not have a matching parameter'
))
if unmatched:
warnings.append(
ValidationWarning(
'UNMATCHED_PARAMETERS',
'No matching filters for these parameters: \'%s\' ' %
unmatched))
return warnings
