def _get_workspace_map(self, configuration): """Builds a mapping for a workspace and configuration of name to model instance. :param configuration: A list of configurations that specify system names of models to fetch and map. :type configuration: list[dict] :returns: A mapping of workspace system names to associated model instances. :rtype: dict[string, :class:`storage.models.Workspace`] """ # Build a mapping of required workspaces results = { file_dict['workspace_name']: None for file_dict in configuration } for workspace in Workspace.objects.filter(name__in=results.keys()): if not workspace.is_active: raise InvalidStrikeConfiguration( 'Workspace is not active: %s' % workspace.name) results[workspace.name] = workspace # Check for any missing workspace model declarations for name, workspace in results.iteritems(): if not workspace: raise InvalidStrikeConfiguration( 'Unknown workspace reference: %s' % name) return results
def __init__(self, configuration, do_validate=False): """Creates a Strike configuration object from the given dictionary. The general format is checked for correctness, but the specified workspace(s) are not checked. :param configuration: The Strike configuration :type configuration: dict :raises InvalidStrikeConfiguration: If the given configuration is invalid """ self._configuration = configuration # Convert old versions if 'version' in self._configuration and self._configuration[ 'version'] == '1.0': raise InvalidStrikeConfiguration( 'Invalid Strike configuration. Strike configuration version 1.0 is no longer supported' ) if 'version' in self._configuration and self._configuration[ 'version'] == '2.0': self._configuration['version'] = SCHEMA_VERSION try: if do_validate: validate(configuration, STRIKE_CONFIGURATION_SCHEMA) except ValidationError as ex: raise InvalidStrikeConfiguration( 'Invalid Strike configuration: %s' % unicode(ex)) self._populate_default_values() if self._configuration['version'] not in SCHEMA_VERSIONS: msg = 'Invalid Strike configuration: %s is an unsupported version number' raise InvalidStrikeConfiguration(msg % self._configuration['version']) self._file_handler = FileHandler() for file_dict in self._configuration['files_to_ingest']: try: regex_pattern = re.compile(file_dict['filename_regex']) except re.error: raise InvalidStrikeConfiguration( 'Invalid file name regex: %s' % file_dict['filename_regex']) new_workspace = None if 'new_workspace' in file_dict: new_workspace = file_dict['new_workspace'] new_file_path = None if 'new_file_path' in file_dict: if os.path.isabs(file_dict['new_file_path']): msg = 'Invalid Strike configuration: new_file_path may not be an absolute path' raise InvalidStrikeConfiguration(msg) file_dict['new_file_path'] = os.path.normpath( file_dict['new_file_path']) new_file_path = file_dict['new_file_path'] rule = FileRule(regex_pattern, file_dict['data_types'], new_workspace, new_file_path) self._file_handler.add_rule(rule)
def __init__(self, configuration): """Creates a Strike configuration object from the given dictionary. The general format is checked for correctness, but the specified workspace(s) are not checked. :param configuration: The Strike configuration :type configuration: dict :raises InvalidStrikeConfiguration: If the given configuration is invalid """ self._configuration = configuration try: validate(configuration, STRIKE_CONFIGURATION_SCHEMA) except ValidationError as ex: raise InvalidStrikeConfiguration( 'Invalid Strike configuration: %s' % unicode(ex)) self._populate_default_values() if not self._configuration['version'] == '1.0': msg = 'Invalid Strike configuration: %s is an unsupported version number' raise InvalidStrikeConfiguration(msg % self._configuration['version']) # Normalize paths for file_dict in self._configuration['files_to_ingest']: if os.path.isabs(file_dict['workspace_path']): msg = 'Invalid Strike configuration: workspace_path may not be an absolute path' raise InvalidStrikeConfiguration(msg) file_dict['workspace_path'] = os.path.normpath( file_dict['workspace_path']) # Build a mapping of required workspaces self._workspace_map = self._get_workspace_map( self._configuration['files_to_ingest']) # Compile and organize regular expressions self.file_regex_entries = [ ] # List of (regex pattern, list of data types, workspace path, workspace) for file_dict in self._configuration['files_to_ingest']: try: regex_pattern = re.compile(file_dict['filename_regex']) except re.error: raise InvalidStrikeConfiguration( 'Invalid file name regex: %s' % file_dict['filename_regex']) regex_tuple = (regex_pattern, file_dict['data_types'], file_dict['workspace_path'], self._workspace_map[file_dict['workspace_name']]) self.file_regex_entries.append(regex_tuple)
def validate(self): """Validates the Strike configuration :returns: A list of warnings discovered during validation :rtype: list[:class:`ingest.strike.configuration.strike_configuration.ValidationWarning`] :raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is invalid. """ warnings = [] monitor_type = self._configuration['monitor']['type'] if monitor_type not in factory.get_monitor_types(): raise InvalidStrikeConfiguration( '\'%s\' is an invalid monitor type' % monitor_type) monitored_workspace_name = self._configuration['workspace'] workspace_names = {monitored_workspace_name} for rule in self._file_handler.rules: if rule.new_workspace: workspace_names.add(rule.new_workspace) for workspace in Workspace.objects.filter(name__in=workspace_names): if workspace.name == monitored_workspace_name: broker_type = workspace.get_broker().broker_type monitor = factory.get_monitor(monitor_type) if broker_type not in monitor.supported_broker_types: msg = 'Monitor type %s does not support broker type %s' raise InvalidStrikeConfiguration( msg % (monitor_type, broker_type)) if not workspace.is_active: raise InvalidStrikeConfiguration( 'Workspace is not active: %s' % workspace.name) workspace_names.remove(workspace.name) if workspace_names: raise InvalidStrikeConfiguration('Unknown workspace name: %s' % workspace_names.pop()) return warnings
def validate(self): """Validates the Strike configuration :returns: A list of warnings discovered during validation :rtype: list[:class:`util.validation.ValidationWarning`] :raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is invalid. """ warnings = [] monitor_type = self.configuration['monitor']['type'] if monitor_type not in factory.get_monitor_types(): raise InvalidStrikeConfiguration( '\'%s\' is an invalid monitor type' % monitor_type) if 'recipe' in self.configuration: recipe_name = self.configuration['recipe'][ 'name'] if 'name' in self.configuration['recipe'] else None revision_num = self.configuration['recipe'][ 'revision_num'] if 'revision_num' in self.configuration[ 'recipe'] else None if not recipe_name: msg = 'Recipe Type name is not defined' raise InvalidStrikeConfiguration(msg) if RecipeType.objects.filter(name=recipe_name).count() == 0: msg = 'Recipe Type %s does not exist' raise InvalidStrikeConfiguration(msg % recipe_name) if revision_num: rt = RecipeType.objects.get(name=recipe_name) if RecipeTypeRevision.objects.filter( recipe_type=rt, revision_num=revision_num).count() == 0: msg = 'Recipe Type revision number %s does not exist for recipe type %s' raise InvalidStrikeConfiguration( msg % (revision_num, recipe_name)) monitored_workspace_name = self.configuration['workspace'] workspace_names = {monitored_workspace_name} for rule in self.file_handler.rules: if rule.new_workspace: workspace_names.add(rule.new_workspace) for workspace in Workspace.objects.filter(name__in=workspace_names): if workspace.name == monitored_workspace_name: broker_type = workspace.get_broker().broker_type monitor = factory.get_monitor(monitor_type) if broker_type not in monitor.supported_broker_types: msg = 'Monitor type %s does not support broker type %s' raise InvalidStrikeConfiguration( msg % (monitor_type, broker_type)) if not workspace.is_active: raise InvalidStrikeConfiguration( 'Workspace is not active: %s' % workspace.name) workspace_names.remove(workspace.name) if workspace_names: raise InvalidStrikeConfiguration('Unknown workspace name: %s' % workspace_names.pop()) return warnings