def _get_workspace_map(self, configuration):
"""Builds a mapping for a workspace and configuration of name to model instance.
:param configuration: A list of configurations that specify system names of models to fetch and map.
:type configuration: list[dict]
:returns: A mapping of workspace system names to associated model instances.
:rtype: dict[string, :class:`storage.models.Workspace`]
"""
# Build a mapping of required workspaces
results = {
file_dict['workspace_name']: None
for file_dict in configuration
}
for workspace in Workspace.objects.filter(name__in=results.keys()):
if not workspace.is_active:
raise InvalidStrikeConfiguration(
'Workspace is not active: %s' % workspace.name)
results[workspace.name] = workspace
# Check for any missing workspace model declarations
for name, workspace in results.iteritems():
if not workspace:
raise InvalidStrikeConfiguration(
'Unknown workspace reference: %s' % name)
return results
def __init__(self, configuration, do_validate=False):
"""Creates a Strike configuration object from the given dictionary. The general format is checked for
correctness, but the specified workspace(s) are not checked.
:param configuration: The Strike configuration
:type configuration: dict
:raises InvalidStrikeConfiguration: If the given configuration is invalid
"""
self._configuration = configuration
# Convert old versions
if 'version' in self._configuration and self._configuration[
'version'] == '1.0':
raise InvalidStrikeConfiguration(
'Invalid Strike configuration. Strike configuration version 1.0 is no longer supported'
)
if 'version' in self._configuration and self._configuration[
'version'] == '2.0':
self._configuration['version'] = SCHEMA_VERSION
try:
if do_validate:
validate(configuration, STRIKE_CONFIGURATION_SCHEMA)
except ValidationError as ex:
raise InvalidStrikeConfiguration(
'Invalid Strike configuration: %s' % unicode(ex))
self._populate_default_values()
if self._configuration['version'] not in SCHEMA_VERSIONS:
msg = 'Invalid Strike configuration: %s is an unsupported version number'
raise InvalidStrikeConfiguration(msg %
self._configuration['version'])
self._file_handler = FileHandler()
for file_dict in self._configuration['files_to_ingest']:
try:
regex_pattern = re.compile(file_dict['filename_regex'])
except re.error:
raise InvalidStrikeConfiguration(
'Invalid file name regex: %s' %
file_dict['filename_regex'])
new_workspace = None
if 'new_workspace' in file_dict:
new_workspace = file_dict['new_workspace']
new_file_path = None
if 'new_file_path' in file_dict:
if os.path.isabs(file_dict['new_file_path']):
msg = 'Invalid Strike configuration: new_file_path may not be an absolute path'
raise InvalidStrikeConfiguration(msg)
file_dict['new_file_path'] = os.path.normpath(
file_dict['new_file_path'])
new_file_path = file_dict['new_file_path']
rule = FileRule(regex_pattern, file_dict['data_types'],
new_workspace, new_file_path)
self._file_handler.add_rule(rule)
def __init__(self, configuration):
"""Creates a Strike configuration object from the given dictionary. The general format is checked for
correctness, but the specified workspace(s) are not checked.
:param configuration: The Strike configuration
:type configuration: dict
:raises InvalidStrikeConfiguration: If the given configuration is invalid
"""
self._configuration = configuration
try:
validate(configuration, STRIKE_CONFIGURATION_SCHEMA)
except ValidationError as ex:
raise InvalidStrikeConfiguration(
'Invalid Strike configuration: %s' % unicode(ex))
self._populate_default_values()
if not self._configuration['version'] == '1.0':
msg = 'Invalid Strike configuration: %s is an unsupported version number'
raise InvalidStrikeConfiguration(msg %
self._configuration['version'])
# Normalize paths
for file_dict in self._configuration['files_to_ingest']:
if os.path.isabs(file_dict['workspace_path']):
msg = 'Invalid Strike configuration: workspace_path may not be an absolute path'
raise InvalidStrikeConfiguration(msg)
file_dict['workspace_path'] = os.path.normpath(
file_dict['workspace_path'])
# Build a mapping of required workspaces
self._workspace_map = self._get_workspace_map(
self._configuration['files_to_ingest'])
# Compile and organize regular expressions
self.file_regex_entries = [
] # List of (regex pattern, list of data types, workspace path, workspace)
for file_dict in self._configuration['files_to_ingest']:
try:
regex_pattern = re.compile(file_dict['filename_regex'])
except re.error:
raise InvalidStrikeConfiguration(
'Invalid file name regex: %s' %
file_dict['filename_regex'])
regex_tuple = (regex_pattern, file_dict['data_types'],
file_dict['workspace_path'],
self._workspace_map[file_dict['workspace_name']])
self.file_regex_entries.append(regex_tuple)
def validate(self):
"""Validates the Strike configuration
:returns: A list of warnings discovered during validation
:rtype: list[:class:`ingest.strike.configuration.strike_configuration.ValidationWarning`]
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
warnings = []
monitor_type = self._configuration['monitor']['type']
if monitor_type not in factory.get_monitor_types():
raise InvalidStrikeConfiguration(
'\'%s\' is an invalid monitor type' % monitor_type)
monitored_workspace_name = self._configuration['workspace']
workspace_names = {monitored_workspace_name}
for rule in self._file_handler.rules:
if rule.new_workspace:
workspace_names.add(rule.new_workspace)
for workspace in Workspace.objects.filter(name__in=workspace_names):
if workspace.name == monitored_workspace_name:
broker_type = workspace.get_broker().broker_type
monitor = factory.get_monitor(monitor_type)
if broker_type not in monitor.supported_broker_types:
msg = 'Monitor type %s does not support broker type %s'
raise InvalidStrikeConfiguration(
msg % (monitor_type, broker_type))
if not workspace.is_active:
raise InvalidStrikeConfiguration(
'Workspace is not active: %s' % workspace.name)
workspace_names.remove(workspace.name)
if workspace_names:
raise InvalidStrikeConfiguration('Unknown workspace name: %s' %
workspace_names.pop())
return warnings
def validate(self):
"""Validates the Strike configuration
:returns: A list of warnings discovered during validation
:rtype: list[:class:`util.validation.ValidationWarning`]
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
warnings = []
monitor_type = self.configuration['monitor']['type']
if monitor_type not in factory.get_monitor_types():
raise InvalidStrikeConfiguration(
'\'%s\' is an invalid monitor type' % monitor_type)
if 'recipe' in self.configuration:
recipe_name = self.configuration['recipe'][
'name'] if 'name' in self.configuration['recipe'] else None
revision_num = self.configuration['recipe'][
'revision_num'] if 'revision_num' in self.configuration[
'recipe'] else None
if not recipe_name:
msg = 'Recipe Type name is not defined'
raise InvalidStrikeConfiguration(msg)
if RecipeType.objects.filter(name=recipe_name).count() == 0:
msg = 'Recipe Type %s does not exist'
raise InvalidStrikeConfiguration(msg % recipe_name)
if revision_num:
rt = RecipeType.objects.get(name=recipe_name)
if RecipeTypeRevision.objects.filter(
recipe_type=rt,
revision_num=revision_num).count() == 0:
msg = 'Recipe Type revision number %s does not exist for recipe type %s'
raise InvalidStrikeConfiguration(
msg % (revision_num, recipe_name))
monitored_workspace_name = self.configuration['workspace']
workspace_names = {monitored_workspace_name}
for rule in self.file_handler.rules:
if rule.new_workspace:
workspace_names.add(rule.new_workspace)
for workspace in Workspace.objects.filter(name__in=workspace_names):
if workspace.name == monitored_workspace_name:
broker_type = workspace.get_broker().broker_type
monitor = factory.get_monitor(monitor_type)
if broker_type not in monitor.supported_broker_types:
msg = 'Monitor type %s does not support broker type %s'
raise InvalidStrikeConfiguration(
msg % (monitor_type, broker_type))
if not workspace.is_active:
raise InvalidStrikeConfiguration(
'Workspace is not active: %s' % workspace.name)
workspace_names.remove(workspace.name)
if workspace_names:
raise InvalidStrikeConfiguration('Unknown workspace name: %s' %
workspace_names.pop())
return warnings