def _validate_settings(self, manifest): """Validates the setting configuration against the given Seed manifest :param manifest: The Seed manifest :type manifest: :class:`job.seed.manifest.SeedManifest` :returns: A list of warnings discovered during validation :rtype: list :raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid """ warnings = [] seed_settings = { setting_dict['name'] for setting_dict in manifest.get_settings() } for name in self.settings: if name not in seed_settings: warnings.append( ValidationWarning('UNKNOWN_SETTING', 'Unknown setting \'%s\' ignored' % name)) for name in seed_settings: if name not in self.settings: warnings.append( ValidationWarning( 'MISSING_SETTING', 'Missing configuration for setting \'%s\'' % name)) return warnings
def _validate_mounts(self, manifest): """Validates the mount configuration against the given Seed manifest :param manifest: The Seed manifest :type manifest: :class:`job.seed.manifest.SeedManifest` :returns: A list of warnings discovered during validation :rtype: list :raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid """ warnings = [] seed_mounts = { mount_dict['name'] for mount_dict in manifest.get_mounts() } for mount_config in self.mounts.values(): warnings.extend(mount_config.validate()) if mount_config.name not in seed_mounts: warnings.append( ValidationWarning( 'UNKNOWN_MOUNT', 'Unknown mount \'%s\' ignored' % mount_config.name)) for name in seed_mounts: if name not in self.mounts: warnings.append( ValidationWarning( 'MISSING_MOUNT', 'Missing configuration for mount \'%s\'' % name)) return warnings
def validate_configuration(self, configuration): """See :meth:`ingest.strike.monitors.monitor.Monitor.validate_configuration` """ warnings = [] if 'sqs_name' not in configuration: raise InvalidMonitorConfiguration('sqs_name is required for s3 monitor') if not isinstance(configuration['sqs_name'], basestring): raise InvalidMonitorConfiguration('sqs_name must be a string') if not configuration['sqs_name']: raise InvalidMonitorConfiguration('sqs_name must be a non-empty string') # If credentials exist, validate them. credentials = AWSClient.instantiate_credentials_from_config(configuration) region_name = configuration.get('region_name') # Check whether the queue can actually be accessed with SQSClient(credentials, region_name) as client: try: client.get_queue_by_name(configuration['sqs_name']) except ClientError: warnings.append(ValidationWarning('sqs_access', 'Unable to access SQS. Check the name, region and credentials.')) return warnings
def _validate_output_workspaces(self, manifest): """Validates the workspace configuration against the given Seed manifest. This will perform database queries. :param manifest: The Seed manifest :type manifest: :class:`job.seed.manifest.SeedManifest` :returns: A list of warnings discovered during validation :rtype: list :raises :class:`job.configuration.exceptions.InvalidJobConfiguration`: If the configuration is invalid """ warnings = [] seed_outputs = { output_dict['name'] for output_dict in manifest.get_outputs()['files'] } seed_outputs.update({ output_dict['name'] for output_dict in manifest.get_outputs()['files'] }) workspace_names = set(self.output_workspaces.values()) if self.default_output_workspace: workspace_names.add(self.default_output_workspace) workspace_models = { w.name: w for w in Workspace.objects.get_workspaces(names=workspace_names) } for name in workspace_names: if name not in workspace_models: raise InvalidJobConfiguration( 'INVALID_WORKSPACE', 'No workspace named \'%s\'' % name) if not workspace_models[name].is_active: warnings.append( ValidationWarning('DEPRECATED_WORKSPACE', 'Workspace \'%s\' is deprecated' % name)) # TODO: add RO/RW mode to workspaces and add warning if using a RO workspace for output if not self.default_output_workspace: for name in seed_outputs: if name not in self.output_workspaces: msg = 'Missing workspace for output \'%s\'' warnings.append( ValidationWarning('MISSING_WORKSPACE', msg % name)) return warnings
def validate_configuration(self, configuration): """See :meth:`ingest.strike.monitors.monitor.Monitor.validate_configuration` """ warnings = [] if 'transfer_suffix' in configuration: if not isinstance(configuration['transfer_suffix'], basestring): raise InvalidMonitorConfiguration('transfer_suffix must be a string') if not configuration['transfer_suffix']: raise InvalidMonitorConfiguration('transfer_suffix must be a non-empty string') if 'transfer_suffix' not in configuration: warnings.append(ValidationWarning('missing_transfer_suffix', 'transfer_suffix is not specified. Using default value of "_tmp"')) return warnings
def validate_configuration(self, config): """See :meth:`storage.brokers.broker.Broker.validate_configuration`""" warnings = [] if 'bucket_name' not in config or not config['bucket_name']: raise InvalidBrokerConfiguration('INVALID_BROKER', 'S3 broker requires "bucket_name" to be populated') region_name = config.get('region_name') credentials = AWSClient.instantiate_credentials_from_config(config) # Check whether the bucket can actually be accessed with S3Client(credentials, region_name) as client: try: client.get_bucket(config['bucket_name']) except (ClientError, NoCredentialsError): warnings.append(ValidationWarning('bucket_access', 'Unable to access bucket. Check the bucket name and credentials.')) return warnings
def validate_connection(self, connecting_parameter): """See :meth:`data.interface.parameter.Parameter.validate_connection` """ warnings = super(FileParameter, self).validate_connection(connecting_parameter) if not self.multiple and connecting_parameter.multiple: msg = 'Parameter \'%s\' cannot accept multiple files' % self.name raise InvalidInterfaceConnection('NO_MULTIPLE_FILES', msg) mismatched_media_types = [] for media_type in connecting_parameter.media_types: if media_type not in self.media_types: mismatched_media_types.append(media_type) if mismatched_media_types: msg = 'Parameter \'%s\' might not accept [%s]' % (self.name, ', '.join(mismatched_media_types)) warnings.append(ValidationWarning('MISMATCHED_MEDIA_TYPES', msg)) return warnings
def validate(self, interface): """Validates this data filter against the given interface :param interface: The interface describing the data that will be passed to the filter :type interface: :class:`data.interface.interface.Interface` :returns: A list of warnings discovered during validation :rtype: :func:`list` :raises :class:`data.filter.exceptions.InvalidDataFilter`: If the data filter is invalid """ warnings = [] unmatched = interface.parameters.keys() for f in self.filter_list: name = f['name'] filter_type = f['type'] if name in interface.parameters: if name in unmatched: unmatched.remove(name) if interface.parameters[ name].param_type == 'file' and filter_type not in FILE_TYPES: raise InvalidDataFilter( 'MISMATCHED_TYPE', 'Interface parameter is a file type and requires a file type filter.' ) if interface.parameters[ name].param_type == 'json' and filter_type in FILE_TYPES: raise InvalidDataFilter( 'MISMATCHED_TYPE', 'Interface parameter is a json type and will not work with a file type filter.' ) if interface.parameters[name].param_type == 'json': if interface.parameters[ name].json_type in STRING_TYPES and filter_type not in STRING_TYPES: raise InvalidDataFilter( 'MISMATCHED_TYPE', 'Interface parameter is a string and filter is not a string type filter' ) if interface.parameters[ name].json_type in NUMBER_TYPES and filter_type not in NUMBER_TYPES: raise InvalidDataFilter( 'MISMATCHED_TYPE', 'Interface parameter is a number and filter is not a number type filter' ) if interface.parameters[ name].json_type in BOOL_TYPES and filter_type not in BOOL_TYPES: raise InvalidDataFilter( 'MISMATCHED_TYPE', 'Interface parameter is a number and filter is not a number type filter' ) json_type = interface.parameters[name].json_type if json_type not in BOOL_TYPES and json_type not in STRING_TYPES and json_type not in NUMBER_TYPES: raise InvalidDataFilter( 'MISMATCHED_TYPE', 'Interface parameter type is not supported by data filters' ) else: warnings.append( ValidationWarning( 'UNMATCHED_FILTER', 'Filter with name \'%s\' does not have a matching parameter' )) if unmatched: warnings.append( ValidationWarning( 'UNMATCHED_PARAMETERS', 'No matching filters for these parameters: \'%s\' ' % unmatched)) return warnings