def test_add_to_zip_copy_function(request, mocker, ymlparsersSetup,
ymlparsersCleanup, exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
split_dirname = 'ymlparsers'
pck = '.'.join(['tests_data', __package__, 'ymlparsers'])
with path(pck, 'config.yml') as full_path:
with tempfile.TemporaryDirectory() as temp_dir:
local_folder = Path(temp_dir)
zip_path = local_folder / "my.zip"
with zipfile.ZipFile(zip_path, mode="w") as zf:
# not real copy, add to zip archive
shutil.copytree(str(full_path.parent),
str(local_folder / "xxxirrelevantxxx"),
ignore=shutil.ignore_patterns(
'__pycache__', 'venv', 'logs', '*.pyc',
'tmp*', '.*', "__init__.py"),
copy_function=add_to_zip_copy_function(
split_dirname=split_dirname, zf=zf),
symlinks=True,
ignore_dangling_symlinks=True)
with zipfile.ZipFile(zip_path, mode="r") as zf:
entry_path = (Path(split_dirname) / 'config.yml').as_posix()
entry_content_bytes = zf.read(str(entry_path))
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load(
[entry_content_bytes.decode('utf-8')])
check_config_yml(default_d, exp_config_d)
def test_ymlparsers_load_multiple_no_substition(request, mocker,
ymlparsersSetup,
ymlparsersCleanup,
exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
exp_d = copy.deepcopy(exp_config_d)
exp_d['general']['log']['formatters']['detail'] = {'format': \
'%(asctime)-14s %(levelname)s [%(name)s.%(funcName)s] %(message)s',
'datefmt': \
'%Y-%m-%d %H:%M:%S'}
exp_d['general']['log']['root']['level'] = 'INFO'
exp_d['app']['inner_host_name'] = 'yahoo.com'
exp_d['app']['white_list'] = ['gamma', 'alpha', 'betha']
exp_d['app']['alt_white_list'] = ['c', 'b', 'a']
pck = '.'.join(['tests_data', __package__, 'ymlparsers'])
with path(pck, 'config.yml') as full_path:
with path(pck, 'config-dev.yml') as full_dev_path:
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load(
[str(full_path), str(full_dev_path)])
app_d = default_d.get('app', None)
exp_app_d = exp_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' in cli_template)
pytest.assume(exp_d == default_d)
def test_ymlparsers_load_single_no_substition(request, mocker, ymlparsersSetup,
ymlparsersCleanup, exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
pck = '.'.join(['tests_data', __package__, 'ymlparsers'])
mock_lock = create_mock_lock(mocker)
mocker.patch.object(HiYaPyCo, 'jinja2Lock', new=mock_lock, spec_set=True)
with path(pck, 'config.yml') as full_path:
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(full_path)])
# ymlparsers.load() when uses HiYaPyCo.jinja2ctx uses HiYaPyCo.jinja2Lock
# note, that ymlparsers.DisableVarSubst also use 1 time HiYaPyCo.jinja2Lock
pytest.assume(mock_lock.acquire.call_count > 1)
pytest.assume(mock_lock.release.call_count == mock_lock.acquire.call_count)
app_d = default_d.get('app', None)
exp_d = copy.deepcopy(exp_config_d)
exp_app_d = exp_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' in cli_template)
pytest.assume(exp_config_d == default_d)
def load_config(argumentParser=None, args=None):
"""
Simplified method for parsing yml configuration file with optionally overrided profiles only.
See alexber.utils.init_app_conf.parse_config() for another variant.
:param argumentParser with instruction how to interpret args. If None, the default one will be instantiated.
:param args: if not None will be used instead of sys.argv
:return:
"""
if ymlparsers.HiYaPyCo.jinja2ctx is None:
raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None")
params, sys_d = parse_sys_args(argumentParser, args)
config_file = params.config_file
full_path = Path(config_file).resolve() # relative to cwd
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(full_path)])
profiles = merge_list_value_in_dicts(sys_d, default_d, conf.GENERAL_KEY,
conf.PROFILES_KEY)
b = is_empty(profiles)
if not b:
# merge to default_d
general_d = default_d.setdefault(conf.GENERAL_KEY, OrderedDict())
general_d[conf.PROFILES_KEY] = profiles
return full_path, default_d
def _run_without_substition(content, exp_config_d, stop):
for i in range(stop):
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(content)])
app_d = default_d.get('app', None)
exp_app_d = exp_config_d.get('app', None)
inner_host_name = app_d.get('inner_host_name', None)
exp_host_name = exp_app_d.get('inner_host_name', None)
pytest.assume(exp_host_name == inner_host_name)
cli_template = app_d.get('cli_template')
pytest.assume('inner_host_name' in cli_template)
pytest.assume(exp_config_d == default_d)
def test_safe_dump(request, mocker, ymlparsersSetup, ymlparsersCleanup,
exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
orig_d = dict(exp_config_d)
d = OrderedDict(exp_config_d)
d['general'] = OrderedDict(d['general'])
d['app'] = OrderedDict(d['app'])
with io.StringIO() as buf:
ymlparsers.safe_dump(d, stream=buf)
value = buf.getvalue()
with ymlparsers.DisableVarSubst():
restored_d = ymlparsers.load(value)
pytest.assume(orig_d == restored_d)
def test_safe_dump_composite(request, mocker, ymlparsersSetup,
ymlparsersCleanup, data, kwds):
logger.info(f'{request._pyfuncitem.name}()')
default_flow_style = kwds.get('default_flow_style', False)
with io.StringIO() as buf:
ymlparsers.safe_dump(data, stream=buf, **kwds)
value = buf.getvalue()
logger.debug(value)
if default_flow_style is not None and not default_flow_style:
pytest.assume('-' in value)
else:
pytest.assume('[' in value)
pytest.assume(']' in value)
with ymlparsers.DisableVarSubst():
restored_d = ymlparsers.load(value)
pytest.assume(data == restored_d)
def test_exception_in_exit(self, request, mocker, ymlparsersSetup,
ymlparsersCleanup, exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
#I check 2 things
#1. Explicately passing params
#2. Releasing lock even if exception occures in DisableVarSubst. __exit__
mock_lock = create_mock_lock(mocker)
orig_jinja2ctx = HiYaPyCo.jinja2ctx
duumy_jinja2ctx = TestDisableVarSubst.DummyEnvironment(
delegate=orig_jinja2ctx)
#orig_exit = ymlparsers.DisableVarSubst.__exit__
# mock_exit = mocker.patch.object(ymlparsers.DisableVarSubst, '__exit__', side_effect=orig_exit,
# autospec=True, spec_set=True)
with pytest.raises(ValueError):
with ymlparsers.DisableVarSubst(jinja2ctx=duumy_jinja2ctx,
jinja2Lock=mock_lock):
pass
# pytest.assume(mock_exit.call_count == 1)
pytest.assume(mock_lock.acquire.call_count > 0)
pytest.assume(
mock_lock.release.call_count == mock_lock.acquire.call_count)
pytest.assume(duumy_jinja2ctx != orig_jinja2ctx)
pytest.assume(mock_lock != orig_jinja2ctx)
pytest.assume(duumy_jinja2ctx.variable_start_string ==
orig_jinja2ctx.variable_start_string)
pytest.assume(duumy_jinja2ctx.variable_end_string ==
orig_jinja2ctx.variable_end_string)
pytest.assume(duumy_jinja2ctx.block_start_string ==
orig_jinja2ctx.block_start_string)
pytest.assume(
duumy_jinja2ctx.block_end_string != orig_jinja2ctx.block_end_string
)
def test_as_str(request, mocker, ymlparsersSetup, ymlparsersCleanup,
exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
orig_d = dict(exp_config_d)
d = OrderedDict(exp_config_d)
d['general'] = OrderedDict(d['general'])
d['app'] = OrderedDict(d['app'])
orig_safe_dump = ymlparsers.safe_dump
mock_safe_dump = mocker.patch.object(ymlparsers,
'safe_dump',
side_effect=orig_safe_dump,
autospec=True,
spec_set=True)
value = ymlparsers.as_str(d)
pytest.assume(mock_safe_dump.call_count > 0)
with ymlparsers.DisableVarSubst():
restored_d = ymlparsers.load(value)
pytest.assume(orig_d == restored_d)
def _parse_sys_args(self, argumentParser=None, args=None):
if ymlparsers.HiYaPyCo.jinja2ctx is None:
raise ValueError("ymlparsers.HiYaPyCo.jinja2ctx can't be None")
params, sys_d0 = self.basic_parse_sys_args(argumentParser, args)
config_file = params.config_file
full_path = Path(config_file).resolve() # relative to cwd
with ymlparsers.DisableVarSubst():
default_d = ymlparsers.load([str(full_path)])
white_list = self._parse_white_list(default_d)
sys_d = self._get_white_listed(sys_d0, white_list)
profiles = self._parse_profiles(sys_d, default_d)
list_ensure = self._parse_list_ensure(sys_d, default_d)
self._do_ensure_list(sys_d, list_ensure)
dd = self.to_convex_map(sys_d)
return dd, profiles, white_list, list_ensure, full_path
def test_exception_in_code(self, request, mocker, ymlparsersSetup,
ymlparsersCleanup, exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
#I check 2 things
#1. Explicately passing params
#2. Releasing lock even if exception occures in the with
mock_lock = create_mock_lock(mocker)
orig_jinja2ctx = HiYaPyCo.jinja2ctx
mocker.patch.object(HiYaPyCo,
'jinja2Lock',
new=mock_lock,
spec_set=True)
jinja2ctx_mock = mocker.patch.object(HiYaPyCo,
'jinja2ctx',
spec_set=True)
mock_variable_start_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.variable_start_string)
type(jinja2ctx_mock).variable_start_string = mock_variable_start_string
mock_variable_end_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.variable_end_string)
type(jinja2ctx_mock).variable_end_string = mock_variable_end_string
mock_block_start_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.block_start_string)
type(jinja2ctx_mock).block_start_string = mock_block_start_string
mock_block_end_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.block_end_string)
type(jinja2ctx_mock).block_end_string = mock_block_end_string
mocks = [
mock_variable_start_string, mock_variable_end_string,
mock_block_start_string, mock_block_end_string
]
# orig_exit = ymlparsers.DisableVarSubst.__exit__
# mock_exit = mocker.patch.object(ymlparsers.DisableVarSubst, '__exit__', side_effect=orig_exit,
# autospec=True, spec_set=True)
with pytest.raises(ValueError):
with ymlparsers.DisableVarSubst():
raise ValueError
# pytest.assume(mock_exit.call_count == 1)
pytest.assume(mock_lock.acquire.call_count > 0)
pytest.assume(
mock_lock.release.call_count == mock_lock.acquire.call_count)
setter_called = None
for mock in mocks:
setter_called = None
pytest.assume(mock.call_count > 0)
for kall in mock.call_args_list:
# (param,), _ = kall
args, _ = kall
if not is_empty(args) and '|' in args[0]:
setter_called = True
break
pytest.assume(setter_called)
pytest.assume(jinja2ctx_mock.variable_start_string ==
orig_jinja2ctx.variable_start_string)
pytest.assume(jinja2ctx_mock.variable_end_string ==
orig_jinja2ctx.variable_end_string)
pytest.assume(jinja2ctx_mock.block_start_string ==
orig_jinja2ctx.block_start_string)
pytest.assume(
jinja2ctx_mock.block_end_string == orig_jinja2ctx.block_end_string)
def test_intented_usage(self, request, mocker, ymlparsersSetup,
ymlparsersCleanup, exp_config_d):
logger.info(f'{request._pyfuncitem.name}()')
mock_lock = create_mock_lock(mocker)
orig_jinja2ctx = HiYaPyCo.jinja2ctx
mocker.patch.object(HiYaPyCo,
'jinja2Lock',
new=mock_lock,
spec_set=True)
jinja2ctx_mock = mocker.patch.object(HiYaPyCo,
'jinja2ctx',
spec_set=True)
mock_variable_start_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.variable_start_string)
type(jinja2ctx_mock).variable_start_string = mock_variable_start_string
mock_variable_end_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.variable_end_string)
type(jinja2ctx_mock).variable_end_string = mock_variable_end_string
mock_block_start_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.block_start_string)
type(jinja2ctx_mock).block_start_string = mock_block_start_string
mock_block_end_string = mocker.PropertyMock(
return_value=orig_jinja2ctx.block_end_string)
type(jinja2ctx_mock).block_end_string = mock_block_end_string
mocks = [
mock_variable_start_string, mock_variable_end_string,
mock_block_start_string, mock_block_end_string
]
# orig_exit = ymlparsers.DisableVarSubst.__exit__
# mock_exit = mocker.patch.object(ymlparsers.DisableVarSubst, '__exit__', side_effect=orig_exit,
# autospec=True, spec_set=True)
with ymlparsers.DisableVarSubst():
logger.debug("dff")
pass
# pytest.assume(mock_exit.call_count == 1)
pytest.assume(mock_lock.acquire.call_count > 0)
pytest.assume(
mock_lock.release.call_count == mock_lock.acquire.call_count)
setter_called = None
for mock in mocks:
setter_called = None
pytest.assume(mock.call_count > 0)
for kall in mock.call_args_list:
#(param,), _ = kall
args, _ = kall
if not is_empty(args) and '|' in args[0]:
setter_called = True
break
pytest.assume(setter_called)
pytest.assume(jinja2ctx_mock.variable_start_string ==
orig_jinja2ctx.variable_start_string)
pytest.assume(jinja2ctx_mock.variable_end_string ==
orig_jinja2ctx.variable_end_string)
pytest.assume(jinja2ctx_mock.block_start_string ==
orig_jinja2ctx.block_start_string)
pytest.assume(
jinja2ctx_mock.block_end_string == orig_jinja2ctx.block_end_string)