def structure(data, location):
# Validate the action type first, so we can use it for other tests
_ = SchemaCheck(
data,
Schema(valid_action(), extra=True),
'action type',
location,
).result()
# Build a valid schema knowing that the action has already been validated
retval = valid_action()
retval.update({
Optional('description', default='No description given'):
Any(str, *string_types)
})
retval.update(
{Optional('options', default=settings.default_options()): dict})
action = data['action']
if action in ['cluster_routing', 'create_index', 'rollover']:
# The cluster_routing, create_index, and rollover actions should not
# have a 'filters' block
pass
elif action == 'alias':
# The alias action should not have a filters block, but should have
# an add and/or remove block.
retval.update({
Optional('add'): dict,
Optional('remove'): dict,
})
else:
retval.update(
{Optional('filters', default=settings.default_filters()): list})
return Schema(retval)
def config_override(ctx, config_dict):
"""Override file-based and default config options with command-line provided ones"""
if config_dict is None:
config_dict = {}
for k in ['client', 'logging']:
if not k in config_dict:
config_dict[k] = {}
for k in list(ctx.params.keys()):
if k in ['dry_run', 'config']:
pass
elif k == 'host':
if 'host' in ctx.params and ctx.params['host'] is not None:
config_dict['client']['hosts'] = ctx.params[k]
elif k in ['loglevel', 'logfile', 'logformat', 'ecs']:
if k in ctx.params and ctx.params[k] is not None:
config_dict['logging'][k] = ctx.params[k]
else:
if k in ctx.params and ctx.params[k] is not None:
config_dict['client'][k] = ctx.params[k]
# After override, prune the nones
for k in ['client', 'logging']:
config_dict[k] = prune_nones(config_dict[k])
return SchemaCheck(config_dict, config_file.client(),
'Client Configuration',
'full configuration dictionary').result()
def filter_schema_check(action, filter_dict):
valid_filters = SchemaCheck(
filter_dict,
Schema(filters.Filters(action, location='singleton')),
'filters',
'{0} singleton action "filters"'.format(action)
).result()
return validate_filters(action, valid_filters)
def option_schema_check(action, option_dict):
clean_options = SchemaCheck(
prune_nones(option_dict),
options.get_schema(action),
'options',
'{0} singleton action "options"'.format(action)
).result()
return prune_excluded(clean_options)
def check_filters(self, filter_dict, loc='singleton', key='filters'):
try:
self.logger.debug(
'Validating provided filters: {0}'.format(filter_dict))
_ = SchemaCheck(
filter_dict, Schema(filters.Filters(self.action,
location=loc)), key,
'{0} singleton action "{1}"'.format(self.action,
key)).result()
self.filters = validate_filters(self.action, _)
except ConfigurationError as e:
self.logger.critical('Unable to parse filters: {0}'.format(e))
sys.exit(1)
def test_config(config):
# Get config from yaml file
yaml_config = get_yaml(config)
# if the file is empty, which is still valid yaml, set as an empty dict
yaml_config = {} if not yaml_config else prune_nones(yaml_config)
# Voluptuous can't verify the schema of a dict if it doesn't have keys,
# so make sure the keys are at least there and are dict()
for k in ['client', 'logging']:
if k not in yaml_config:
yaml_config[k] = {}
else:
yaml_config[k] = prune_nones(yaml_config[k])
return SchemaCheck(yaml_config, config_file.client(),
'Client Configuration', 'full configuration dictionary').result()
def iterate_filters(self, config):
"""
Iterate over the filters defined in `config` and execute them.
:arg config: A dictionary of filters, as extracted from the YAML
configuration file.
.. note:: `config` should be a dictionary with the following form:
.. code-block:: python
{ 'filters' : [
{
'filtertype': 'the_filter_type',
'key1' : 'value1',
...
'keyN' : 'valueN'
}
]
}
"""
# Make sure we actually _have_ filters to act on
if not 'filters' in config or len(config['filters']) < 1:
self.loggit.info('No filters in config. Returning unaltered object.')
return
self.loggit.debug('All filters: {0}'.format(config['filters']))
for f in config['filters']:
self.loggit.debug('Top of the loop: {0}'.format(self.snapshots))
self.loggit.debug('Un-parsed filter args: {0}'.format(f))
self.loggit.debug('Parsed filter args: {0}'.format(
SchemaCheck(
f,
filters.structure(),
'filter',
'SnapshotList.iterate_filters'
).result()
)
)
method = self.__map_method(f['filtertype'])
# Remove key 'filtertype' from dictionary 'f'
del f['filtertype']
# If it's a filtertype with arguments, update the defaults with the
# provided settings.
self.loggit.debug('Filter args: {0}'.format(f))
self.loggit.debug('Pre-instance: {0}'.format(self.snapshots))
method(**f)
self.loggit.debug('Post-instance: {0}'.format(self.snapshots))
def f(v):
def prune_nones(mydict):
return dict([(k, v) for k, v in mydict.items()
if v != None and v != 'None'])
# This validator method simply validates all filters in the list.
for idx in range(0, len(v)):
pruned = prune_nones(v[idx])
filter_dict = SchemaCheck(
pruned, single(action, pruned), 'filter',
'{0}, filter #{1}: {2}'.format(location, idx,
pruned)).result()
logger.debug('Filter #{0}: {1}'.format(idx, filter_dict))
v[idx] = filter_dict
# If we've made it here without raising an Exception, it's valid
return v
def check_options(self, option_dict):
try:
self.logger.debug(
'Validating provided options: {0}'.format(option_dict))
# Kludgy work-around to needing 'repository' in options for these actions
# but only to pass the schema check. It's removed again below.
if self.action in ['delete_snapshots', 'restore']:
option_dict['repository'] = self.repository
_ = SchemaCheck(
prune_nones(option_dict), options.get_schema(self.action),
'options',
'{0} singleton action "options"'.format(self.action)).result()
self.options = self.prune_excluded(_)
# Remove this after the schema check, as the action class won't need it as an arg
if self.action in ['delete_snapshots', 'restore']:
del self.options['repository']
except ConfigurationError as e:
self.logger.critical('Unable to parse options: {0}'.format(e))
sys.exit(1)
def filter_schema_check(action, filter_dict):
"""Validate the provided filters against the filter schema"""
valid_filters = SchemaCheck(
filter_dict, Schema(filters.Filters(action, location='singleton')),
'filters', '{0} singleton action "filters"'.format(action)).result()
return validate_filters(action, valid_filters)
def option_schema_check(action, option_dict):
"""Validate command-line options against the option schema"""
clean_options = SchemaCheck(
prune_nones(option_dict), options.get_schema(action), 'options',
'{0} singleton action "options"'.format(action)).result()
return prune_excluded(clean_options)