def test_ymlparsers_load_multiple_no_substition(request, mocker,
ymlparsersSetup,
ymlparsersCleanup,
exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
exp_d = copy.deepcopy(exp_config_d)
exp_d['general']['log']['formatters']['detail'] = {'format': \
'%(asctime)-14s %(levelname)s [%(name)s.%(funcName)s] %(message)s',
'datefmt': \
'%Y-%m-%d %H:%M:%S'}
exp_d['general']['log']['root']['level'] = 'INFO'
exp_d['app']['inner_host_name'] = 'yahoo.com'
exp_d['app']['white_list'] = ['gamma', 'alpha', 'betha']
exp_d['app']['alt_white_list'] = ['c', 'b', 'a']
pck = '.'.join(['tests_data', __package__, 'ymlparsers'])
with path(pck, 'config.yml') as full_path:
with path(pck, 'config-dev.yml') as full_dev_path:
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load(
[str(full_path), str(full_dev_path)])
app_d = default_d.get('app', None)
exp_app_d = exp_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' in cli_template)
pytest.assume(exp_d == default_d)
def test_ymlparsers_load_single_no_substition(request, mocker, ymlparsersSetup,
ymlparsersCleanup, exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
pck = '.'.join(['tests_data', __package__, 'ymlparsers'])
mock_lock = create_mock_lock(mocker)
mocker.patch.object(HiYaPyCo, 'jinja2Lock', new=mock_lock, spec_set=True)
with path(pck, 'config.yml') as full_path:
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(full_path)])
# ymlparsers.load() when uses HiYaPyCo.jinja2ctx uses HiYaPyCo.jinja2Lock
# note, that ymlparsers.DisableVarSubst also use 1 time HiYaPyCo.jinja2Lock
pytest.assume(mock_lock.acquire.call_count > 1)
pytest.assume(mock_lock.release.call_count == mock_lock.acquire.call_count)
app_d = default_d.get('app', None)
exp_d = copy.deepcopy(exp_config_d)
exp_app_d = exp_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' in cli_template)
pytest.assume(exp_config_d == default_d)
def test_add_to_zip_copy_function(request, mocker, ymlparsersSetup,
ymlparsersCleanup, exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
split_dirname = 'ymlparsers'
pck = '.'.join(['tests_data', __package__, 'ymlparsers'])
with path(pck, 'config.yml') as full_path:
with tempfile.TemporaryDirectory() as temp_dir:
local_folder = Path(temp_dir)
zip_path = local_folder / "my.zip"
with zipfile.ZipFile(zip_path, mode="w") as zf:
# not real copy, add to zip archive
shutil.copytree(str(full_path.parent),
str(local_folder / "xxxirrelevantxxx"),
ignore=shutil.ignore_patterns(
'__pycache__', 'venv', 'logs', '*.pyc',
'tmp*', '.*', "__init__.py"),
copy_function=add_to_zip_copy_function(
split_dirname=split_dirname, zf=zf),
symlinks=True,
ignore_dangling_symlinks=True)
with zipfile.ZipFile(zip_path, mode="r") as zf:
entry_path = (Path(split_dirname) / 'config.yml').as_posix()
entry_content_bytes = zf.read(str(entry_path))
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load(
[entry_content_bytes.decode('utf-8')])
check_config_yml(default_d, exp_config_d)
def _parse_yml(self, sys_d, profiles, config_file='config.yml'):
if ymlparsers.HiYaPyCo.jinja2ctx is None:
raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None")
base_full_path = Path(config_file).resolve() # relative to cwd
base_suffix = base_full_path.suffix
base_name = base_full_path.with_suffix("").name
with _io.StringIO() as sys_buf:
ymlparsers.safe_dump(sys_d, sys_buf)
buffersize = len(profiles) + 1 # default_profile
yml_files = deque(maxlen=buffersize)
yml_files.append(str(base_full_path))
full_path = base_full_path
for profile in profiles:
name = base_name + '-' + profile + base_suffix
full_path = full_path.with_name(name)
yml_files.append(str(full_path))
dd = ymlparsers.load([*yml_files, sys_buf.getvalue()])
return dd
def load_config(argumentParser=None, args=None):
"""
Simplified method for parsing yml configuration file with optionally overrided profiles only.
See alexber.utils.init_app_conf.parse_config() for another variant.
:param argumentParser with instruction how to interpret args. If None, the default one will be instantiated.
:param args: if not None will be used instead of sys.argv
:return:
"""
if ymlparsers.HiYaPyCo.jinja2ctx is None:
raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None")
params, sys_d = parse_sys_args(argumentParser, args)
config_file = params.config_file
full_path = Path(config_file).resolve() # relative to cwd
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(full_path)])
profiles = merge_list_value_in_dicts(sys_d, default_d, conf.GENERAL_KEY,
conf.PROFILES_KEY)
b = is_empty(profiles)
if not b:
# merge to default_d
general_d = default_d.setdefault(conf.GENERAL_KEY, OrderedDict())
general_d[conf.PROFILES_KEY] = profiles
return full_path, default_d
def _run_without_substition(content, exp_config_d, stop):
for i in range(stop):
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(content)])
app_d = default_d.get('app', None)
exp_app_d = exp_config_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' in cli_template)
pytest.assume(exp_config_d == default_d)
def test_safe_dump(request, mocker, ymlparsersSetup, ymlparsersCleanup,
exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
orig_d = dict(exp_config_d)
d = OrderedDict(exp_config_d)
d['general'] = OrderedDict(d['general'])
d['app'] = OrderedDict(d['app'])
with io.StringIO() as buf:
ymlparsers.safe_dump(d, stream=buf)
value = buf.getvalue()
with ymlparsers.DisableVarSubst():
restored_d = ymlparsers.load(value)
pytest.assume(orig_d == restored_d)
def test_safe_dump_composite(request, mocker, ymlparsersSetup,
ymlparsersCleanup, data, kwds):
logger.info(f'{request._pyfuncitem.name}()')
default_flow_style = kwds.get('default_flow_style', False)
with io.StringIO() as buf:
ymlparsers.safe_dump(data, stream=buf, **kwds)
value = buf.getvalue()
logger.debug(value)
if default_flow_style is not None and not default_flow_style:
pytest.assume('-' in value)
else:
pytest.assume('[' in value)
pytest.assume(']' in value)
with ymlparsers.DisableVarSubst():
restored_d = ymlparsers.load(value)
pytest.assume(data == restored_d)
def test_ymlparsers_load_single_with_substition(request, mocker,
ymlparsersSetup,
ymlparsersCleanup,
exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
pck = '.'.join(['tests_data', __package__, 'ymlparsers'])
mock_lock = create_mock_lock(mocker)
mocker.patch.object(HiYaPyCo, 'jinja2Lock', new=mock_lock, spec_set=True)
with path(pck, 'config.yml') as full_path:
default_d = ymlparsers.load([str(full_path)])
pytest.assume(mock_lock.acquire.call_count > 1)
pytest.assume(mock_lock.release.call_count == mock_lock.acquire.call_count)
app_d = default_d['app']
exp_d = copy.deepcopy(exp_config_d)
exp_app_d = exp_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
exp_cli_template = exp_app_d.get('cli_template', None)
exp_cli_template = format_template(exp_cli_template,
app_inner_host_name=exp_host_name)
exp_app_d['cli_template'] = exp_cli_template
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' not in cli_template)
cli_template = app_d['cli_template']
pytest.assume('inner_host_name' not in cli_template)
pytest.assume(exp_d == default_d)
white_list = app_d.get('white_list', None)
exp_white_list = exp_app_d.get('white_list', None)
pytest.assume(exp_white_list == white_list)
def _run_with_substition(content, exp_config_d, stop):
for i in range(stop):
default_d = ymlparsers.load([str(content)])
app_d = default_d['app']
exp_app_d = exp_config_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
exp_cli_template = exp_app_d.get('cli_template', None)
exp_cli_template = format_template(exp_cli_template,
app_inner_host_name=exp_host_name)
exp_app_d['cli_template'] = exp_cli_template
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' not in cli_template)
cli_template = app_d['cli_template']
pytest.assume('inner_host_name' not in cli_template)
pytest.assume(exp_config_d == default_d)
def test_as_str(request, mocker, ymlparsersSetup, ymlparsersCleanup,
exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
orig_d = dict(exp_config_d)
d = OrderedDict(exp_config_d)
d['general'] = OrderedDict(d['general'])
d['app'] = OrderedDict(d['app'])
orig_safe_dump = ymlparsers.safe_dump
mock_safe_dump = mocker.patch.object(ymlparsers,
'safe_dump',
side_effect=orig_safe_dump,
autospec=True,
spec_set=True)
value = ymlparsers.as_str(d)
pytest.assume(mock_safe_dump.call_count > 0)
with ymlparsers.DisableVarSubst():
restored_d = ymlparsers.load(value)
pytest.assume(orig_d == restored_d)
def _parse_sys_args(self, argumentParser=None, args=None):
if ymlparsers.HiYaPyCo.jinja2ctx is None:
raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None")
params, sys_d0 = self.basic_parse_sys_args(argumentParser, args)
config_file = params.config_file
full_path = Path(config_file).resolve() # relative to cwd
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(full_path)])
white_list = self._parse_white_list(default_d)
sys_d = self._get_white_listed(sys_d0, white_list)
profiles = self._parse_profiles(sys_d, default_d)
list_ensure = self._parse_list_ensure(sys_d, default_d)
self._do_ensure_list(sys_d, list_ensure)
dd = self.to_convex_map(sys_d)
return dd, profiles, white_list, list_ensure, full_path