def reconfigure(ns=namespace, **kwargs):
"""Reconfigures the given kwargs, restoring the current configuration for
only those kwargs when the contextmanager exits.
Args:
ns: Namespace of the conf
"""
conf_namespace = staticconf.config.get_namespace(ns)
starting_config = {
k: v
for k, v in conf_namespace.get_config_values().iteritems()
if k in kwargs
}
staticconf.DictConfiguration(kwargs, namespace=ns)
try:
yield
finally:
final_config = {
k: v
for k, v in conf_namespace.get_config_values().iteritems()
if k not in kwargs
}
final_config.update(starting_config)
staticconf.config.get_namespace(ns).clear()
staticconf.DictConfiguration(final_config, namespace=ns)
def test_reload_default(self):
staticconf.DictConfiguration(dict(one='three', seven='nine'))
one, seven = staticconf.get('one'), staticconf.get('seven')
staticconf.DictConfiguration(dict(one='ten', seven='el'))
staticconf.reload()
assert_equal(one, 'ten')
assert_equal(seven, 'el')
def test_load_and_validate_namespace(self):
real_config = {'SomeClass.min': 20, 'SomeClass.max': 25}
staticconf.DictConfiguration(self.config)
staticconf.DictConfiguration(real_config, namespace='real')
some_class = SomeClass()
assert_equal(some_class.max, 100)
assert_equal(some_class.min, 0)
assert_equal(some_class.real_min, 20)
assert_equal(some_class.real_max, 25)
def test_validate_all_passes(self):
name = 'yan'
staticconf.DictConfiguration({}, namespace=name)
staticconf.DictConfiguration({})
config.validate(all_names=True)
staticconf.get_string('one.two')
staticconf.get_string('foo', namespace=name)
staticconf.DictConfiguration({'one.two': 'nice'})
staticconf.DictConfiguration({'foo': 'nice'}, namespace=name)
config.validate(all_names=True)
def test_reload_single(self):
name = 'another_one'
staticconf.DictConfiguration(dict(one='three'))
staticconf.DictConfiguration(dict(two='three'), namespace=name)
one, two = staticconf.get('one'), staticconf.get('two', namespace=name)
# access the values to set the value_proxy cache
one.value, two.value
staticconf.DictConfiguration(dict(one='four'))
staticconf.DictConfiguration(dict(two='five'), namespace=name)
staticconf.reload()
assert_equal(one, 'four')
assert_equal(two, 'three')
def configure_from_dict(config_dict):
"""Configure the :mod:`data_pipeline` clientlib from a dictionary.
Args:
config_dict (dict): a dict of config data
"""
staticconf.DictConfiguration(config_dict, namespace=namespace)
def main(args: argparse.Namespace) -> None:
staticconf.DictConfiguration({'yes': args.yes})
destination, destination_str = _parse_destination(args.dest, args.name)
before_timestamp = parse_time(args.before) if args.before else int(
time.time())
backup_store = get_backup_store(args.name)
with backup_store.unlock(preserve_scratch=args.preserve_scratch_dir):
files_to_restore: List[ManifestEntry]
if args.sha:
files_to_restore = backup_store.manifest.get_entries_by_sha(
args.sha)
if not files_to_restore:
raise ValueError(
f'Sha {args.sha} does not match anything in the store')
else:
search_results = backup_store.manifest.search(
like=args.like,
before_timestamp=before_timestamp,
history_limit=1,
)
# Restore the most recent version of all files that haven't been deleted
files_to_restore = [h[0] for _, h in search_results if h[0].sha]
if _confirm_restore(files_to_restore, destination, destination_str):
_restore(files_to_restore, destination, backup_store)
def load_yaml(file_path) -> dict:
if file_path not in staticconf.config.configuration_namespaces:
yaml_dict = load_dict_from_yaml(file_path)
staticconf.DictConfiguration(yaml_dict,
namespace=file_path,
flatten=False)
return staticconf.config.configuration_namespaces[
file_path].configuration_values
def setup_config(args: argparse.Namespace) -> None:
# load_default_config merges the 'module_config' key from the first file
# and the 'module_env_config' key from the second file to configure packages.
# This allows us to configure packages differently in different hiera envs by
# changing 'module_env_config'. We use the same file for both keys.
_load_module_configs(args.env_config_path)
signals_branch_or_tag = getattr(args, 'signals_branch_or_tag', None)
cluster_config_directory = getattr(args, 'cluster_config_directory',
None) or DEFAULT_CLUSTER_DIRECTORY
staticconf.DictConfiguration(
{'cluster_config_directory': cluster_config_directory})
aws_region = getattr(args, 'aws_region', None)
cluster = getattr(args, 'cluster', None)
pool = getattr(args, 'pool', None)
scheduler = getattr(args, 'scheduler', None)
if aws_region and cluster:
raise argparse.ArgumentError(
None, 'Cannot specify both cluster and aws_region')
# If there is a cluster specified via --cluster, load cluster-specific attributes
# into staticconf. These values are not specified using hiera in srv-configs because
# we might want to be operating on a cluster in one region while running from a
# different region.
elif cluster:
aws_region = staticconf.read_string(f'clusters.{cluster}.aws_region',
default=None)
if pool:
load_cluster_pool_config(cluster, pool, scheduler,
signals_branch_or_tag)
staticconf.DictConfiguration({'aws': {'region': aws_region}})
boto_creds_file = staticconf.read_string('aws.access_key_file',
default=None)
if boto_creds_file:
staticconf.JSONConfiguration(boto_creds_file,
namespace=CREDENTIALS_NAMESPACE)
if signals_branch_or_tag:
staticconf.DictConfiguration(
{'autoscale_signal': {
'branch_or_tag': signals_branch_or_tag
}})
def load_variables(file_path, namespace):
file_path = Template(file_path).render(os.environ)
variables = dict(
Configuration.from_file(os.path.abspath(file_path)).configure()
)
staticconf.DictConfiguration(
variables,
namespace=namespace,
flatten=False)
def load_yaml(file_path, namespace):
file_path = Template(file_path).render(os.environ)
yaml_dict = Configuration.from_file(
os.path.abspath(file_path),
configure=False
)
staticconf.DictConfiguration(
yaml_dict,
namespace=namespace,
flatten=False)
def load_cluster_pool_config(cluster: str, pool: str, scheduler: str,
signals_branch_or_tag: Optional[str]) -> None:
pool_namespace = POOL_NAMESPACE.format(pool=pool, scheduler=scheduler)
pool_config_file = get_pool_config_path(cluster, pool, scheduler)
staticconf.YamlConfiguration(pool_config_file, namespace=pool_namespace)
if signals_branch_or_tag:
staticconf.DictConfiguration(
{'autoscale_signal': {
'branch_or_tag': signals_branch_or_tag
}},
namespace=pool_namespace,
)
def test_load_and_validate(self):
staticconf.DictConfiguration(self.config)
some_class = SomeClass()
assert_equal(some_class.max, 100)
assert_equal(some_class.min, 0)
assert_equal(some_class.ratio, 7.7)
assert_equal(some_class.alt_ratio, 6.0)
assert_equal(staticconf.get('globals'), False)
assert_equal(staticconf.get('enable'), 'True')
assert_equal(staticconf.get_bool('enable'), True)
assert_equal(some_class.msg, None)
assert staticconf.get_regex('matcher').match('12345')
assert not staticconf.get_regex('matcher').match('a')
assert_equal(staticconf.get_list_of_int('options'), [1, 7, 3, 9])
def _load_module_configs(env_config_path: str):
staticconf.YamlConfiguration(env_config_path)
for config in staticconf.read_list('module_config', default=[]):
if 'file' in config:
staticconf.YamlConfiguration(config['file'],
namespace=config['namespace'])
staticconf.DictConfiguration(config.get('config', {}),
namespace=config['namespace'])
if 'initialize' in config:
path = config['initialize'].split('.')
function = path.pop()
module_name = '.'.join(path)
module = __import__(module_name, globals(), locals(), [path[-1]])
getattr(module, function)()
def main(args):
args.start_time = parse_time_string(args.start_time)
args.end_time = parse_time_string(args.end_time)
staticconf.DictConfiguration({
'join_delay_mean_seconds':
args.join_delay_params[0],
'join_delay_stdev_seconds':
args.join_delay_params[1],
'cpus_per_weight':
args.cpus_per_weight,
'ebs_volume_size':
args.ebs_volume_size,
})
# We can provide up to two simulation objects to compare. If we load two simulator objects to compare,
# we don't need to run a simulation here. If the user specifies --compare but only gives one object,
# then we need to run a simulation now, and use that to compare to the saved sim
sims = []
if args.compare:
if len(args.compare) > 2:
raise argparse.ArgumentError(
None,
f'Cannot compare more than two simulations: {args.compare}')
sims = [
read_object_from_compressed_json(sim_file)
for sim_file in args.compare
]
if len(sims) < 2:
metrics_client = _load_metrics(args.metrics_data_files, args.pool)
simulator = _run_simulation(args, metrics_client)
sims.insert(0, simulator)
if len(sims) == 2:
cmp_fn = getattr(operator, args.comparison_operator)
final_simulator = cmp_fn(*sims)
else:
final_simulator = sims[0]
if args.simulation_result_file:
write_object_to_compressed_json(final_simulator,
args.simulation_result_file)
if hasattr(args, 'reports'):
if 'all' in args.reports:
args.reports = REPORT_TYPES.keys()
for report in args.reports:
make_report(report, final_simulator, args.start_time,
args.end_time, args.output_prefix)
def main(args: argparse.Namespace) -> None:
staticconf.YamlConfiguration(args.config, flatten=False)
backup_set_config = staticconf.read('backups')[args.name]
staticconf.DictConfiguration(backup_set_config, namespace=args.name)
backup_store = get_backup_store(args.name)
if args.manifest:
manifest = Manifest(args.filename)
private_key_filename = backup_store.config.read('private_key_filename',
default='')
lock_manifest(
manifest,
private_key_filename,
backup_store._save,
backup_store._load,
backup_store.options,
)
else:
with backup_store.unlock():
backup_store.save_if_new(args.filename)
def main(args: argparse.Namespace) -> None:
staticconf.DictConfiguration({'yes': args.yes})
backup_store = get_backup_store(args.name)
with backup_store.unlock(preserve_scratch=args.preserve_scratch_dir):
files_to_verify: List[ManifestEntry]
if args.fast:
_fast_verify(backup_store)
return
elif args.sha:
files_to_verify = backup_store.manifest.get_entries_by_sha(
args.sha)
if not files_to_verify:
raise ValueError(
f'Sha {args.sha} does not match anything in the store')
else:
search_results = backup_store.manifest.search(like=args.like, )
# Verify the most recent version of all files that haven't been deleted
files_to_verify = [h[0] for _, h in search_results if h[0].sha]
_verify(files_to_verify, backup_store, args.show_all)
def configure_packages(configs, flatten=True, log_keys_only=True):
"""Load configuration into a :class:`staticconf.config.ConfigNamespace`
and optionally call an initialize function after loading configuration
for a package.
`configs` should be a `meta-config`, which is a list of configuration
sections. Each section **must** have a `namespace` key and *may* have
one or more other keys:
namespace
the name of a :class:`staticconf.config.ConfigNamespace` which will hold
the configs loaded from a `file` or `config` in this section
file
the path to a yaml file. The contents of this file are loaded
using :func:`staticconf.loader.YamlConfiguration`
config
a yaml mapping structure. The contents of this mapping are loaded
using :func:`staticconf.loader.DictConfiguration`
For each section in the `meta-config` the following operations are
performed:
1. Load the contents of `file`, if the key is present
2. Load the contents of `config`, if the key is present. If values
were loaded from `file`, these new `config` values will override
previous values.
Example configuration:
.. code-block:: yaml
module_config:
- namespace: prod_ns
config:
access_log_name: tmp_service_<service_name>
error_log_name: tmp_service_error_<service_name>
- namespace: memcache
config:
clients:
encoders:
service_prefix: '<service_name>_service'
Usage:
.. code-block:: python
configs = {...} # The contents of the example configuration above
configure_packages(configs['module_config'])
:param configs: List of config dicts.
:param flatten: boolean for whether staticconf should load each module
config with a flattened dictionary. Defaults to True.
:param log_keys_only: boolean for whether staticonf should only log unknown keys
"""
for config in configs or []:
# 1st load a yaml
if 'file' in config:
staticconf.YamlConfiguration(
config['file'],
namespace=config['namespace'],
flatten=flatten,
log_keys_only=log_keys_only,
)
# 2nd update with a config dict
if 'config' in config:
staticconf.DictConfiguration(
config['config'],
namespace=config['namespace'],
flatten=flatten,
log_keys_only=log_keys_only,
)
def test_validate_single_passes(self):
staticconf.DictConfiguration({})
config.validate()
staticconf.get_string('one.two')
staticconf.DictConfiguration({'one.two': 'nice'})
config.validate()
def load_yaml(file_path, namespace):
file_path = Template(file_path).render(os.environ)
yaml_dict = YamlToGo(os.path.abspath(file_path)).yaml
staticconf.DictConfiguration(yaml_dict, namespace=namespace, flatten=False)
def initialize_module_3():
staticconf.DictConfiguration(dict(key='Number 3 lives'),
namespace='module3')
def test_readers(self):
staticconf.DictConfiguration(self.config)
assert_equal(staticconf.read_float('SomeClass.ratio'), 7.7)
assert_equal(staticconf.read_bool('globals'), False)
assert_equal(staticconf.read_list_of_int('options'), [1, 7, 3, 9])
def setup_config(config_file: str) -> None:
staticconf.YamlConfiguration(config_file, flatten=False)
for backup_name, backup_config in staticconf.read('backups').items():
staticconf.DictConfiguration(backup_config, namespace=backup_name)
def config():
staticconf.DictConfiguration(
yaml.load(TEST_CONFIG),
namespace=moneybot.CONFIG_NS,
)
def configure_packages(configs, ignore_initialize=False, flatten=True):
"""Load configuration into a :class:`staticconf.config.ConfigNamespace`
and optionally call an initialize function after loading configuration
for a package.
`configs` should be a `meta-config`, which is a list of configuration
sections. Each section **must** have a `namespace` key and *may* have
one or more other keys:
namespace
the name of a :class:`staticconf.config.ConfigNamespace` which will hold
the configs loaded from a `file` or `config` in this section
file
the path to a yaml file. The contents of this file are loaded
using :func:`staticconf.loader.YamlConfiguration`
config
a yaml mapping structure. The contents of this mapping are loaded
using :func:`staticconf.loader.DictConfiguration`
initialize
a callable which accepts no arguments. It can be used to initialize
state after configuration is loaded.
For each section in the `meta-config` the following operations are
performed:
1. Load the contents of `file`, if the key is present
2. Load the contents of `config`, if the key is present. If values
were loaded from `file`, these new `config` values will override
previous values.
3. Call `initialize`, if the key is present
Example configuration:
.. code-block:: yaml
module_config:
- namespace: yelp_pyramid
config:
access_log_name: tmp_service_<service_name>
error_log_name: tmp_service_error_<service_name>
- namespace: clog
initialize: yelp_servlib.clog_util.initialize
config:
log_stream_name: <service_name>
file:
/nail/srv/configs/clog.yaml
- namespace: yelp_memcache
config:
clients:
encoders:
service_prefix: '<service_name>_service'
Usage:
.. code-block:: python
configs = {...} # The contents of the example configuration above
configure_packages(configs['module_config'])
:param configs: List of config dicts.
:param ignore_initialize: Whether to ignore the `initialize` key in the
config and not call it. This may be used to
reload a config whose initialize is only intended
to be called once. Defaults to `False`.
Available from version >=4.5.4.
:param flatten: boolean for whether staticconf should load each module
config with a flattened dictionary. Defaults to True.
"""
for config in configs or []:
# 1st load a yaml
if 'file' in config:
staticconf.YamlConfiguration(config['file'],
namespace=config['namespace'],
flatten=flatten)
# 2nd update with a config dict
if 'config' in config:
staticconf.DictConfiguration(config['config'],
namespace=config['namespace'],
flatten=flatten)
# 3rd call the initialize method
if not ignore_initialize and 'initialize' in config:
path = config['initialize'].split('.')
function = path.pop()
module_name = '.'.join(path)
module = __import__(str(module_name), globals(), locals(),
[str(path[-1])])
getattr(module, function)()
