def test_ymlparsers_load_multiple_no_substition(request, mocker, ymlparsersSetup, ymlparsersCleanup, exp_config_d): logger.info(f'{request._pyfuncitem.name}()') exp_d = copy.deepcopy(exp_config_d) exp_d['general']['log']['formatters']['detail'] = {'format': \ '%(asctime)-14s %(levelname)s [%(name)s.%(funcName)s] %(message)s', 'datefmt': \ '%Y-%m-%d %H:%M:%S'} exp_d['general']['log']['root']['level'] = 'INFO' exp_d['app']['inner_host_name'] = 'yahoo.com' exp_d['app']['white_list'] = ['gamma', 'alpha', 'betha'] exp_d['app']['alt_white_list'] = ['c', 'b', 'a'] pck = '.'.join(['tests_data', __package__, 'ymlparsers']) with path(pck, 'config.yml') as full_path: with path(pck, 'config-dev.yml') as full_dev_path: with ymlparsers.DisableVarSubst(): default_d = ymlparsers.load( [str(full_path), str(full_dev_path)]) app_d = default_d.get('app', None) exp_app_d = exp_d.get('app', None) inner_host_name = app_d.get('inner_host_name', None) exp_host_name = exp_app_d.get('inner_host_name', None) pytest.assume(exp_host_name == inner_host_name) cli_template = app_d.get('cli_template') pytest.assume('inner_host_name' in cli_template) pytest.assume(exp_d == default_d)
def test_ymlparsers_load_single_no_substition(request, mocker, ymlparsersSetup, ymlparsersCleanup, exp_config_d): logger.info(f'{request._pyfuncitem.name}()') pck = '.'.join(['tests_data', __package__, 'ymlparsers']) mock_lock = create_mock_lock(mocker) mocker.patch.object(HiYaPyCo, 'jinja2Lock', new=mock_lock, spec_set=True) with path(pck, 'config.yml') as full_path: with ymlparsers.DisableVarSubst(): default_d = ymlparsers.load([str(full_path)]) # ymlparsers.load() when uses HiYaPyCo.jinja2ctx uses HiYaPyCo.jinja2Lock # note, that ymlparsers.DisableVarSubst also use 1 time HiYaPyCo.jinja2Lock pytest.assume(mock_lock.acquire.call_count > 1) pytest.assume(mock_lock.release.call_count == mock_lock.acquire.call_count) app_d = default_d.get('app', None) exp_d = copy.deepcopy(exp_config_d) exp_app_d = exp_d.get('app', None) inner_host_name = app_d.get('inner_host_name', None) exp_host_name = exp_app_d.get('inner_host_name', None) pytest.assume(exp_host_name == inner_host_name) cli_template = app_d.get('cli_template') pytest.assume('inner_host_name' in cli_template) pytest.assume(exp_config_d == default_d)
def test_add_to_zip_copy_function(request, mocker, ymlparsersSetup, ymlparsersCleanup, exp_config_d): logger.info(f'{request._pyfuncitem.name}()') split_dirname = 'ymlparsers' pck = '.'.join(['tests_data', __package__, 'ymlparsers']) with path(pck, 'config.yml') as full_path: with tempfile.TemporaryDirectory() as temp_dir: local_folder = Path(temp_dir) zip_path = local_folder / "my.zip" with zipfile.ZipFile(zip_path, mode="w") as zf: # not real copy, add to zip archive shutil.copytree(str(full_path.parent), str(local_folder / "xxxirrelevantxxx"), ignore=shutil.ignore_patterns( '__pycache__', 'venv', 'logs', '*.pyc', 'tmp*', '.*', "__init__.py"), copy_function=add_to_zip_copy_function( split_dirname=split_dirname, zf=zf), symlinks=True, ignore_dangling_symlinks=True) with zipfile.ZipFile(zip_path, mode="r") as zf: entry_path = (Path(split_dirname) / 'config.yml').as_posix() entry_content_bytes = zf.read(str(entry_path)) with ymlparsers.DisableVarSubst(): default_d = ymlparsers.load( [entry_content_bytes.decode('utf-8')]) check_config_yml(default_d, exp_config_d)
def _parse_yml(self, sys_d, profiles, config_file='config.yml'): if ymlparsers.HiYaPyCo.jinja2ctx is None: raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None") base_full_path = Path(config_file).resolve() # relative to cwd base_suffix = base_full_path.suffix base_name = base_full_path.with_suffix("").name with _io.StringIO() as sys_buf: ymlparsers.safe_dump(sys_d, sys_buf) buffersize = len(profiles) + 1 # default_profile yml_files = deque(maxlen=buffersize) yml_files.append(str(base_full_path)) full_path = base_full_path for profile in profiles: name = base_name + '-' + profile + base_suffix full_path = full_path.with_name(name) yml_files.append(str(full_path)) dd = ymlparsers.load([*yml_files, sys_buf.getvalue()]) return dd
def load_config(argumentParser=None, args=None): """ Simplified method for parsing yml configuration file with optionally overrided profiles only. See alexber.utils.init_app_conf.parse_config() for another variant. :param argumentParser with instruction how to interpret args. If None, the default one will be instantiated. :param args: if not None will be used instead of sys.argv :return: """ if ymlparsers.HiYaPyCo.jinja2ctx is None: raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None") params, sys_d = parse_sys_args(argumentParser, args) config_file = params.config_file full_path = Path(config_file).resolve() # relative to cwd with ymlparsers.DisableVarSubst(): default_d = ymlparsers.load([str(full_path)]) profiles = merge_list_value_in_dicts(sys_d, default_d, conf.GENERAL_KEY, conf.PROFILES_KEY) b = is_empty(profiles) if not b: # merge to default_d general_d = default_d.setdefault(conf.GENERAL_KEY, OrderedDict()) general_d[conf.PROFILES_KEY] = profiles return full_path, default_d
def _run_without_substition(content, exp_config_d, stop): for i in range(stop): with ymlparsers.DisableVarSubst(): default_d = ymlparsers.load([str(content)]) app_d = default_d.get('app', None) exp_app_d = exp_config_d.get('app', None) inner_host_name = app_d.get('inner_host_name', None) exp_host_name = exp_app_d.get('inner_host_name', None) pytest.assume(exp_host_name == inner_host_name) cli_template = app_d.get('cli_template') pytest.assume('inner_host_name' in cli_template) pytest.assume(exp_config_d == default_d)
def test_safe_dump(request, mocker, ymlparsersSetup, ymlparsersCleanup, exp_config_d): logger.info(f'{request._pyfuncitem.name}()') orig_d = dict(exp_config_d) d = OrderedDict(exp_config_d) d['general'] = OrderedDict(d['general']) d['app'] = OrderedDict(d['app']) with io.StringIO() as buf: ymlparsers.safe_dump(d, stream=buf) value = buf.getvalue() with ymlparsers.DisableVarSubst(): restored_d = ymlparsers.load(value) pytest.assume(orig_d == restored_d)
def test_safe_dump_composite(request, mocker, ymlparsersSetup, ymlparsersCleanup, data, kwds): logger.info(f'{request._pyfuncitem.name}()') default_flow_style = kwds.get('default_flow_style', False) with io.StringIO() as buf: ymlparsers.safe_dump(data, stream=buf, **kwds) value = buf.getvalue() logger.debug(value) if default_flow_style is not None and not default_flow_style: pytest.assume('-' in value) else: pytest.assume('[' in value) pytest.assume(']' in value) with ymlparsers.DisableVarSubst(): restored_d = ymlparsers.load(value) pytest.assume(data == restored_d)
def test_ymlparsers_load_single_with_substition(request, mocker, ymlparsersSetup, ymlparsersCleanup, exp_config_d): logger.info(f'{request._pyfuncitem.name}()') pck = '.'.join(['tests_data', __package__, 'ymlparsers']) mock_lock = create_mock_lock(mocker) mocker.patch.object(HiYaPyCo, 'jinja2Lock', new=mock_lock, spec_set=True) with path(pck, 'config.yml') as full_path: default_d = ymlparsers.load([str(full_path)]) pytest.assume(mock_lock.acquire.call_count > 1) pytest.assume(mock_lock.release.call_count == mock_lock.acquire.call_count) app_d = default_d['app'] exp_d = copy.deepcopy(exp_config_d) exp_app_d = exp_d.get('app', None) inner_host_name = app_d.get('inner_host_name', None) exp_host_name = exp_app_d.get('inner_host_name', None) pytest.assume(exp_host_name == inner_host_name) exp_cli_template = exp_app_d.get('cli_template', None) exp_cli_template = format_template(exp_cli_template, app_inner_host_name=exp_host_name) exp_app_d['cli_template'] = exp_cli_template cli_template = app_d.get('cli_template') pytest.assume('inner_host_name' not in cli_template) cli_template = app_d['cli_template'] pytest.assume('inner_host_name' not in cli_template) pytest.assume(exp_d == default_d) white_list = app_d.get('white_list', None) exp_white_list = exp_app_d.get('white_list', None) pytest.assume(exp_white_list == white_list)
def _run_with_substition(content, exp_config_d, stop): for i in range(stop): default_d = ymlparsers.load([str(content)]) app_d = default_d['app'] exp_app_d = exp_config_d.get('app', None) inner_host_name = app_d.get('inner_host_name', None) exp_host_name = exp_app_d.get('inner_host_name', None) pytest.assume(exp_host_name == inner_host_name) exp_cli_template = exp_app_d.get('cli_template', None) exp_cli_template = format_template(exp_cli_template, app_inner_host_name=exp_host_name) exp_app_d['cli_template'] = exp_cli_template cli_template = app_d.get('cli_template') pytest.assume('inner_host_name' not in cli_template) cli_template = app_d['cli_template'] pytest.assume('inner_host_name' not in cli_template) pytest.assume(exp_config_d == default_d)
def test_as_str(request, mocker, ymlparsersSetup, ymlparsersCleanup, exp_config_d): logger.info(f'{request._pyfuncitem.name}()') orig_d = dict(exp_config_d) d = OrderedDict(exp_config_d) d['general'] = OrderedDict(d['general']) d['app'] = OrderedDict(d['app']) orig_safe_dump = ymlparsers.safe_dump mock_safe_dump = mocker.patch.object(ymlparsers, 'safe_dump', side_effect=orig_safe_dump, autospec=True, spec_set=True) value = ymlparsers.as_str(d) pytest.assume(mock_safe_dump.call_count > 0) with ymlparsers.DisableVarSubst(): restored_d = ymlparsers.load(value) pytest.assume(orig_d == restored_d)
def _parse_sys_args(self, argumentParser=None, args=None): if ymlparsers.HiYaPyCo.jinja2ctx is None: raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None") params, sys_d0 = self.basic_parse_sys_args(argumentParser, args) config_file = params.config_file full_path = Path(config_file).resolve() # relative to cwd with ymlparsers.DisableVarSubst(): default_d = ymlparsers.load([str(full_path)]) white_list = self._parse_white_list(default_d) sys_d = self._get_white_listed(sys_d0, white_list) profiles = self._parse_profiles(sys_d, default_d) list_ensure = self._parse_list_ensure(sys_d, default_d) self._do_ensure_list(sys_d, list_ensure) dd = self.to_convex_map(sys_d) return dd, profiles, white_list, list_ensure, full_path