def reconfigure(ns=namespace, **kwargs): """Reconfigures the given kwargs, restoring the current configuration for only those kwargs when the contextmanager exits. Args: ns: Namespace of the conf """ conf_namespace = staticconf.config.get_namespace(ns) starting_config = { k: v for k, v in conf_namespace.get_config_values().iteritems() if k in kwargs } staticconf.DictConfiguration(kwargs, namespace=ns) try: yield finally: final_config = { k: v for k, v in conf_namespace.get_config_values().iteritems() if k not in kwargs } final_config.update(starting_config) staticconf.config.get_namespace(ns).clear() staticconf.DictConfiguration(final_config, namespace=ns)
def test_reload_default(self): staticconf.DictConfiguration(dict(one='three', seven='nine')) one, seven = staticconf.get('one'), staticconf.get('seven') staticconf.DictConfiguration(dict(one='ten', seven='el')) staticconf.reload() assert_equal(one, 'ten') assert_equal(seven, 'el')
def test_load_and_validate_namespace(self): real_config = {'SomeClass.min': 20, 'SomeClass.max': 25} staticconf.DictConfiguration(self.config) staticconf.DictConfiguration(real_config, namespace='real') some_class = SomeClass() assert_equal(some_class.max, 100) assert_equal(some_class.min, 0) assert_equal(some_class.real_min, 20) assert_equal(some_class.real_max, 25)
def test_validate_all_passes(self): name = 'yan' staticconf.DictConfiguration({}, namespace=name) staticconf.DictConfiguration({}) config.validate(all_names=True) staticconf.get_string('one.two') staticconf.get_string('foo', namespace=name) staticconf.DictConfiguration({'one.two': 'nice'}) staticconf.DictConfiguration({'foo': 'nice'}, namespace=name) config.validate(all_names=True)
def test_reload_single(self): name = 'another_one' staticconf.DictConfiguration(dict(one='three')) staticconf.DictConfiguration(dict(two='three'), namespace=name) one, two = staticconf.get('one'), staticconf.get('two', namespace=name) # access the values to set the value_proxy cache one.value, two.value staticconf.DictConfiguration(dict(one='four')) staticconf.DictConfiguration(dict(two='five'), namespace=name) staticconf.reload() assert_equal(one, 'four') assert_equal(two, 'three')
def configure_from_dict(config_dict): """Configure the :mod:`data_pipeline` clientlib from a dictionary. Args: config_dict (dict): a dict of config data """ staticconf.DictConfiguration(config_dict, namespace=namespace)
def main(args: argparse.Namespace) -> None: staticconf.DictConfiguration({'yes': args.yes}) destination, destination_str = _parse_destination(args.dest, args.name) before_timestamp = parse_time(args.before) if args.before else int( time.time()) backup_store = get_backup_store(args.name) with backup_store.unlock(preserve_scratch=args.preserve_scratch_dir): files_to_restore: List[ManifestEntry] if args.sha: files_to_restore = backup_store.manifest.get_entries_by_sha( args.sha) if not files_to_restore: raise ValueError( f'Sha {args.sha} does not match anything in the store') else: search_results = backup_store.manifest.search( like=args.like, before_timestamp=before_timestamp, history_limit=1, ) # Restore the most recent version of all files that haven't been deleted files_to_restore = [h[0] for _, h in search_results if h[0].sha] if _confirm_restore(files_to_restore, destination, destination_str): _restore(files_to_restore, destination, backup_store)
def load_yaml(file_path) -> dict: if file_path not in staticconf.config.configuration_namespaces: yaml_dict = load_dict_from_yaml(file_path) staticconf.DictConfiguration(yaml_dict, namespace=file_path, flatten=False) return staticconf.config.configuration_namespaces[ file_path].configuration_values
def setup_config(args: argparse.Namespace) -> None: # load_default_config merges the 'module_config' key from the first file # and the 'module_env_config' key from the second file to configure packages. # This allows us to configure packages differently in different hiera envs by # changing 'module_env_config'. We use the same file for both keys. _load_module_configs(args.env_config_path) signals_branch_or_tag = getattr(args, 'signals_branch_or_tag', None) cluster_config_directory = getattr(args, 'cluster_config_directory', None) or DEFAULT_CLUSTER_DIRECTORY staticconf.DictConfiguration( {'cluster_config_directory': cluster_config_directory}) aws_region = getattr(args, 'aws_region', None) cluster = getattr(args, 'cluster', None) pool = getattr(args, 'pool', None) scheduler = getattr(args, 'scheduler', None) if aws_region and cluster: raise argparse.ArgumentError( None, 'Cannot specify both cluster and aws_region') # If there is a cluster specified via --cluster, load cluster-specific attributes # into staticconf. These values are not specified using hiera in srv-configs because # we might want to be operating on a cluster in one region while running from a # different region. elif cluster: aws_region = staticconf.read_string(f'clusters.{cluster}.aws_region', default=None) if pool: load_cluster_pool_config(cluster, pool, scheduler, signals_branch_or_tag) staticconf.DictConfiguration({'aws': {'region': aws_region}}) boto_creds_file = staticconf.read_string('aws.access_key_file', default=None) if boto_creds_file: staticconf.JSONConfiguration(boto_creds_file, namespace=CREDENTIALS_NAMESPACE) if signals_branch_or_tag: staticconf.DictConfiguration( {'autoscale_signal': { 'branch_or_tag': signals_branch_or_tag }})
def load_variables(file_path, namespace): file_path = Template(file_path).render(os.environ) variables = dict( Configuration.from_file(os.path.abspath(file_path)).configure() ) staticconf.DictConfiguration( variables, namespace=namespace, flatten=False)
def load_yaml(file_path, namespace): file_path = Template(file_path).render(os.environ) yaml_dict = Configuration.from_file( os.path.abspath(file_path), configure=False ) staticconf.DictConfiguration( yaml_dict, namespace=namespace, flatten=False)
def load_cluster_pool_config(cluster: str, pool: str, scheduler: str, signals_branch_or_tag: Optional[str]) -> None: pool_namespace = POOL_NAMESPACE.format(pool=pool, scheduler=scheduler) pool_config_file = get_pool_config_path(cluster, pool, scheduler) staticconf.YamlConfiguration(pool_config_file, namespace=pool_namespace) if signals_branch_or_tag: staticconf.DictConfiguration( {'autoscale_signal': { 'branch_or_tag': signals_branch_or_tag }}, namespace=pool_namespace, )
def test_load_and_validate(self): staticconf.DictConfiguration(self.config) some_class = SomeClass() assert_equal(some_class.max, 100) assert_equal(some_class.min, 0) assert_equal(some_class.ratio, 7.7) assert_equal(some_class.alt_ratio, 6.0) assert_equal(staticconf.get('globals'), False) assert_equal(staticconf.get('enable'), 'True') assert_equal(staticconf.get_bool('enable'), True) assert_equal(some_class.msg, None) assert staticconf.get_regex('matcher').match('12345') assert not staticconf.get_regex('matcher').match('a') assert_equal(staticconf.get_list_of_int('options'), [1, 7, 3, 9])
def _load_module_configs(env_config_path: str): staticconf.YamlConfiguration(env_config_path) for config in staticconf.read_list('module_config', default=[]): if 'file' in config: staticconf.YamlConfiguration(config['file'], namespace=config['namespace']) staticconf.DictConfiguration(config.get('config', {}), namespace=config['namespace']) if 'initialize' in config: path = config['initialize'].split('.') function = path.pop() module_name = '.'.join(path) module = __import__(module_name, globals(), locals(), [path[-1]]) getattr(module, function)()
def main(args): args.start_time = parse_time_string(args.start_time) args.end_time = parse_time_string(args.end_time) staticconf.DictConfiguration({ 'join_delay_mean_seconds': args.join_delay_params[0], 'join_delay_stdev_seconds': args.join_delay_params[1], 'cpus_per_weight': args.cpus_per_weight, 'ebs_volume_size': args.ebs_volume_size, }) # We can provide up to two simulation objects to compare. If we load two simulator objects to compare, # we don't need to run a simulation here. If the user specifies --compare but only gives one object, # then we need to run a simulation now, and use that to compare to the saved sim sims = [] if args.compare: if len(args.compare) > 2: raise argparse.ArgumentError( None, f'Cannot compare more than two simulations: {args.compare}') sims = [ read_object_from_compressed_json(sim_file) for sim_file in args.compare ] if len(sims) < 2: metrics_client = _load_metrics(args.metrics_data_files, args.pool) simulator = _run_simulation(args, metrics_client) sims.insert(0, simulator) if len(sims) == 2: cmp_fn = getattr(operator, args.comparison_operator) final_simulator = cmp_fn(*sims) else: final_simulator = sims[0] if args.simulation_result_file: write_object_to_compressed_json(final_simulator, args.simulation_result_file) if hasattr(args, 'reports'): if 'all' in args.reports: args.reports = REPORT_TYPES.keys() for report in args.reports: make_report(report, final_simulator, args.start_time, args.end_time, args.output_prefix)
def main(args: argparse.Namespace) -> None: staticconf.YamlConfiguration(args.config, flatten=False) backup_set_config = staticconf.read('backups')[args.name] staticconf.DictConfiguration(backup_set_config, namespace=args.name) backup_store = get_backup_store(args.name) if args.manifest: manifest = Manifest(args.filename) private_key_filename = backup_store.config.read('private_key_filename', default='') lock_manifest( manifest, private_key_filename, backup_store._save, backup_store._load, backup_store.options, ) else: with backup_store.unlock(): backup_store.save_if_new(args.filename)
def main(args: argparse.Namespace) -> None: staticconf.DictConfiguration({'yes': args.yes}) backup_store = get_backup_store(args.name) with backup_store.unlock(preserve_scratch=args.preserve_scratch_dir): files_to_verify: List[ManifestEntry] if args.fast: _fast_verify(backup_store) return elif args.sha: files_to_verify = backup_store.manifest.get_entries_by_sha( args.sha) if not files_to_verify: raise ValueError( f'Sha {args.sha} does not match anything in the store') else: search_results = backup_store.manifest.search(like=args.like, ) # Verify the most recent version of all files that haven't been deleted files_to_verify = [h[0] for _, h in search_results if h[0].sha] _verify(files_to_verify, backup_store, args.show_all)
def configure_packages(configs, flatten=True, log_keys_only=True): """Load configuration into a :class:`staticconf.config.ConfigNamespace` and optionally call an initialize function after loading configuration for a package. `configs` should be a `meta-config`, which is a list of configuration sections. Each section **must** have a `namespace` key and *may* have one or more other keys: namespace the name of a :class:`staticconf.config.ConfigNamespace` which will hold the configs loaded from a `file` or `config` in this section file the path to a yaml file. The contents of this file are loaded using :func:`staticconf.loader.YamlConfiguration` config a yaml mapping structure. The contents of this mapping are loaded using :func:`staticconf.loader.DictConfiguration` For each section in the `meta-config` the following operations are performed: 1. Load the contents of `file`, if the key is present 2. Load the contents of `config`, if the key is present. If values were loaded from `file`, these new `config` values will override previous values. Example configuration: .. code-block:: yaml module_config: - namespace: prod_ns config: access_log_name: tmp_service_<service_name> error_log_name: tmp_service_error_<service_name> - namespace: memcache config: clients: encoders: service_prefix: '<service_name>_service' Usage: .. code-block:: python configs = {...} # The contents of the example configuration above configure_packages(configs['module_config']) :param configs: List of config dicts. :param flatten: boolean for whether staticconf should load each module config with a flattened dictionary. Defaults to True. :param log_keys_only: boolean for whether staticonf should only log unknown keys """ for config in configs or []: # 1st load a yaml if 'file' in config: staticconf.YamlConfiguration( config['file'], namespace=config['namespace'], flatten=flatten, log_keys_only=log_keys_only, ) # 2nd update with a config dict if 'config' in config: staticconf.DictConfiguration( config['config'], namespace=config['namespace'], flatten=flatten, log_keys_only=log_keys_only, )
def test_validate_single_passes(self): staticconf.DictConfiguration({}) config.validate() staticconf.get_string('one.two') staticconf.DictConfiguration({'one.two': 'nice'}) config.validate()
def load_yaml(file_path, namespace): file_path = Template(file_path).render(os.environ) yaml_dict = YamlToGo(os.path.abspath(file_path)).yaml staticconf.DictConfiguration(yaml_dict, namespace=namespace, flatten=False)
def initialize_module_3(): staticconf.DictConfiguration(dict(key='Number 3 lives'), namespace='module3')
def test_readers(self): staticconf.DictConfiguration(self.config) assert_equal(staticconf.read_float('SomeClass.ratio'), 7.7) assert_equal(staticconf.read_bool('globals'), False) assert_equal(staticconf.read_list_of_int('options'), [1, 7, 3, 9])
def setup_config(config_file: str) -> None: staticconf.YamlConfiguration(config_file, flatten=False) for backup_name, backup_config in staticconf.read('backups').items(): staticconf.DictConfiguration(backup_config, namespace=backup_name)
def config(): staticconf.DictConfiguration( yaml.load(TEST_CONFIG), namespace=moneybot.CONFIG_NS, )
def configure_packages(configs, ignore_initialize=False, flatten=True): """Load configuration into a :class:`staticconf.config.ConfigNamespace` and optionally call an initialize function after loading configuration for a package. `configs` should be a `meta-config`, which is a list of configuration sections. Each section **must** have a `namespace` key and *may* have one or more other keys: namespace the name of a :class:`staticconf.config.ConfigNamespace` which will hold the configs loaded from a `file` or `config` in this section file the path to a yaml file. The contents of this file are loaded using :func:`staticconf.loader.YamlConfiguration` config a yaml mapping structure. The contents of this mapping are loaded using :func:`staticconf.loader.DictConfiguration` initialize a callable which accepts no arguments. It can be used to initialize state after configuration is loaded. For each section in the `meta-config` the following operations are performed: 1. Load the contents of `file`, if the key is present 2. Load the contents of `config`, if the key is present. If values were loaded from `file`, these new `config` values will override previous values. 3. Call `initialize`, if the key is present Example configuration: .. code-block:: yaml module_config: - namespace: yelp_pyramid config: access_log_name: tmp_service_<service_name> error_log_name: tmp_service_error_<service_name> - namespace: clog initialize: yelp_servlib.clog_util.initialize config: log_stream_name: <service_name> file: /nail/srv/configs/clog.yaml - namespace: yelp_memcache config: clients: encoders: service_prefix: '<service_name>_service' Usage: .. code-block:: python configs = {...} # The contents of the example configuration above configure_packages(configs['module_config']) :param configs: List of config dicts. :param ignore_initialize: Whether to ignore the `initialize` key in the config and not call it. This may be used to reload a config whose initialize is only intended to be called once. Defaults to `False`. Available from version >=4.5.4. :param flatten: boolean for whether staticconf should load each module config with a flattened dictionary. Defaults to True. """ for config in configs or []: # 1st load a yaml if 'file' in config: staticconf.YamlConfiguration(config['file'], namespace=config['namespace'], flatten=flatten) # 2nd update with a config dict if 'config' in config: staticconf.DictConfiguration(config['config'], namespace=config['namespace'], flatten=flatten) # 3rd call the initialize method if not ignore_initialize and 'initialize' in config: path = config['initialize'].split('.') function = path.pop() module_name = '.'.join(path) module = __import__(str(module_name), globals(), locals(), [str(path[-1])]) getattr(module, function)()