def test_strict_parsing():
importlib.reload(expandvars)
with pytest.raises(
expandvars.ExpandvarsException, match="FOO: parameter null or not set"
) as e:
expandvars.expandvars("${FOO:?}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
with pytest.raises(
expandvars.ExpandvarsException, match="FOO: parameter null or not set"
) as e:
expandvars.expandvars("${FOO?}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
with pytest.raises(expandvars.ExpandvarsException, match="FOO: custom error") as e:
expandvars.expandvars("${FOO:?custom error}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
with pytest.raises(expandvars.ExpandvarsException, match="FOO: custom error") as e:
expandvars.expandvars("${FOO?custom error}")
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
env.update({"FOO": "foo"})
assert expandvars.expandvars("${FOO:?custom err}") == "foo"
assert expandvars.expandvars("${FOO?custom err}:bar") == "foo:bar"
def test_expandvars_get_default():
importlib.reload(expandvars)
assert expandvars.expandvars("${FOO-default}") == "default"
assert expandvars.expandvars("${FOO:-default}") == "default"
assert expandvars.expandvars("${FOO:-}") == ""
assert expandvars.expandvars("${FOO:-foo}:${FOO-bar}") == "foo:bar"
def test_expandvars_indirection():
importlib.reload(expandvars)
assert expandvars.expandvars("${!FOO}:${FOO}") == "foo:X"
assert expandvars.expandvars("${!FOO-default}") == "foo"
assert expandvars.expandvars("${!BAR-default}") == "default"
assert expandvars.expandvars("${!X-default}") == "default"
def expandvars_dict(settings: t.Dict[str, str]) -> t.Dict[str, t.Any]:
"""Expand strings or variables from the environment into equivalent python literals or strings.
>>> from pyramid_heroku import expandvars_dict
>>> from pprint import pprint
>>> from unittest import mock
>>> import os
>>> with mock.patch.dict(os.environ,{'STRING':'text', 'BOOL': 'true', 'INT':'1', 'NESTED':'nested_${STRING}'}):
... pprint(expandvars_dict({
... "bool": "true",
... "default_bool": "${FOO:-false}",
... "default_int": "${FOO:-2}",
... "default_string": "${FOO:-bar}",
... "env_bool": "${BOOL:-foo}",
... "env_int": "${INT:-foo}",
... "env_string": "${STRING:-foo}",
... "int": "1",
... "must_be_set": "${STRING:? error: STRING must be set!}",
... "nested": "${NESTED:-foo}",
... "string": "foo",
... }))
{'bool': True,
'default_bool': False,
'default_int': 2,
'default_string': 'bar',
'env_bool': True,
'env_int': 1,
'env_string': 'text',
'int': 1,
'must_be_set': 'text',
'nested': 'nested_text',
'string': 'foo'}
"""
return {k: safe_eval(expandvars(expandvars(v))) for k, v in settings.items()}
def test_invalid_length_err():
importlib.reload(expandvars)
with pytest.raises(expandvars.ExpandvarsException,
match="FOO: -3: substring expression < 0") as e:
expandvars.expandvars("${FOO:1:-3}")
assert isinstance(e.value, expandvars.NegativeSubStringExpression)
def test_offset():
importlib.reload(expandvars)
assert expandvars.expandvars("${FOO:3}") == "nbigfoobar"
assert expandvars.expandvars("${FOO: 4 }") == "bigfoobar"
assert expandvars.expandvars("${FOO:30}") == ""
assert expandvars.expandvars("${FOO:0}") == "damnbigfoobar"
assert expandvars.expandvars("${FOO:-3}:bar") == "damnbigfoobar:bar"
def test_offset():
os.environ.update({"FOO": "damnbigfoobar"})
assert expandvars("${FOO:3}") == "nbigfoobar"
assert expandvars("${FOO: 4}") == "bigfoobar"
assert expandvars("${FOO:30}") == ""
assert expandvars("${FOO:0}") == "damnbigfoobar"
assert expandvars("${FOO:foo}") == "damnbigfoobar"
def test_expandvars_substitute():
if "BAR" in os.environ:
del os.environ["BAR"]
os.environ.update({"FOO": "bar"})
assert expandvars("${FOO:+foo}") == "foo"
assert expandvars("${BAR:+foo}") == ""
assert expandvars("${BAR:+}") == ""
def test_missing_escapped_character():
importlib.reload(expandvars)
with pytest.raises(expandvars.ExpandvarsException) as e:
expandvars.expandvars("$FOO\\")
assert str(e.value) == "$FOO\\: missing escaped character"
assert isinstance(e.value, expandvars.MissingExcapedChar)
def test_strict_parsing_recover_null():
importlib.reload(expandvars)
assert expandvars.expandvars("${FOO:?}:${BAR?}") == "foo:bar"
assert expandvars.expandvars("${FOO:?custom err}:${BAR?custom err}") == "foo:bar"
assert expandvars.expandvars("$FOO$BAR", nounset=True) == "foobar"
assert expandvars.expandvars("${FOO}:${BAR}", nounset=True) == "foo:bar"
def resolve_element(element: str) -> str:
if re.search("(\$\{).+(\})", element): # noqa: W605
return expandvars(element, nounset=True)
elif element.startswith("$"):
try:
return expandvars(element, nounset=True)
except UnboundVariable:
return element
else:
return element
def test_brace_never_closed_err():
os.environ.update({"FOO": "damnbigfoobar"})
with pytest.raises(ValueError) as e:
expandvars("${FOO:")
assert str(e.value) == "${FOO:: '{' was never closed."
with pytest.raises(ValueError) as e:
expandvars("${FOO}${BAR")
assert str(e.value) == "${BAR: '{' was never closed."
def resolve_element(element: str) -> str:
if element.startswith("${") & element.endswith("}"):
return expandvars(element, nounset=True)
elif element.startswith("$"):
try:
return expandvars(element, nounset=True)
except UnboundVariable:
return element
else:
return element
def test_expandvars_update_default():
if "FOO" in os.environ:
del os.environ["FOO"]
assert expandvars("${FOO:=}") == ""
del os.environ["FOO"]
assert expandvars("${FOO:=default}") == "default"
assert os.environ.get("FOO") == "default"
assert expandvars("${FOO:=ignoreme}") == "default"
def test_bad_substitution_err():
importlib.reload(expandvars)
with pytest.raises(expandvars.ExpandvarsException) as e:
expandvars.expandvars("${FOO:}") == ""
assert str(e.value) == "${FOO:}: bad substitution"
assert isinstance(e.value, expandvars.BadSubstitution)
with pytest.raises(expandvars.ExpandvarsException) as e:
expandvars.expandvars("${}") == ""
assert str(e.value) == "${}: bad substitution"
assert isinstance(e.value, expandvars.BadSubstitution)
def test_offset_length():
os.environ.update({"FOO": "damnbigfoobar"})
assert expandvars("${FOO:4:3}") == "big"
assert expandvars("${FOO: 7:6}") == "foobar"
assert expandvars("${FOO:7: 100}") == "foobar"
assert expandvars("${FOO:0:100}") == "damnbigfoobar"
assert expandvars("${FOO:70:10}") == ""
assert expandvars("${FOO:1:0}") == ""
assert expandvars("${FOO:0:}") == ""
assert expandvars("${FOO:0:foo}") == ""
assert expandvars("${FOO::}") == ""
assert expandvars("${FOO::5}") == "damnb"
def test_offset_length():
importlib.reload(expandvars)
assert expandvars.expandvars("${FOO:4:3}") == "big"
assert expandvars.expandvars("${FOO: 7:6 }") == "foobar"
assert expandvars.expandvars("${FOO:7: 100 }") == "foobar"
assert expandvars.expandvars("${FOO:0:100}") == "damnbigfoobar"
assert expandvars.expandvars("${FOO:70:10}") == ""
assert expandvars.expandvars("${FOO:1:0}") == ""
assert expandvars.expandvars("${FOO:0:}") == ""
assert expandvars.expandvars("${FOO::}") == ""
assert expandvars.expandvars("${FOO::5}") == "damnb"
assert expandvars.expandvars("${FOO:-3:1}:bar") == "damnbigfoobar:bar"
def test_strict_parsing():
if "FOO" in os.environ:
del os.environ["FOO"]
with pytest.raises(ValueError) as e:
expandvars("${FOO:?}")
assert str(e.value) == "FOO: parameter null or not set"
with pytest.raises(ValueError) as e:
expandvars("${FOO:?custom error}")
assert str(e.value) == "FOO: custom error"
os.environ.update({"FOO": "foo"})
assert expandvars("${FOO:?custom err}") == "foo"
def get_environ_keys(*args, **kwargs):
from expandvars import expandvars
k = kwargs.get('key')
v = kwargs.get('value')
assert (k is not None) and (
v is not None), 'Problem with kwargs -> {}, k={}, v={}'.format(
kwargs, k, v)
__logger__ = kwargs.get('logger')
if (k == '__LITERALS__'):
_v = eval(v)
if (isinstance(_v, list)):
for item in _v:
env_literals.append(item)
else:
env_literals.append(_v)
if (isinstance(v, str)):
v = expandvars(v) if (k not in env_literals) else v
v = __escape(v) if (k in __env__.get('__ESCAPED__', [])) else v
ignoring = __env__.get('IGNORING', [])
environ = kwargs.get('environ', None)
if (isinstance(environ, dict)):
environ[k] = v
if (k not in ignoring):
__env__[k] = v
if (__logger__):
__logger__.info('\t{} -> {}'.format(k, environ.get(k)))
return tuple([k, v])
def from_string(text, cwd="./"):
# expand environment variables
protected = (text.replace("$/", "~INCLUDE~").replace(
"$.", "~REFERENCE~").replace("$self.", "~SELFREF~"))
parsed = (expandvars(protected).replace("~INCLUDE~", "$/").replace(
"~REFERENCE~", "$.").replace("~SELFREF~", "$self."))
config = yaml.load(parsed, lambda stream: Loader(stream, cwd))
# process includes
for key, include in config.pop("includes", {}).items():
if not isinstance(include, dict):
raise ValueError(
f"Include '{key}' must be a mapping. {type(include)} given.")
for k, v in include.items():
if k not in ["mixins", "components"
] and not k.startswith("components:"):
continue
if k not in config:
config[k] = v
continue
config[k].extend(v)
return config
def test_expandvars_option_nounset():
importlib.reload(expandvars)
assert expandvars.expandvars("$FOO") == ""
with pytest.raises(expandvars.ExpandvarsException,
match="FOO: unbound variable") as e:
expandvars.expandvars("$FOO", nounset=True)
assert isinstance(e.value, expandvars.UnboundVariable)
with pytest.raises(expandvars.ExpandvarsException,
match="FOO: unbound variable") as e:
expandvars.expandvars("${FOO}", nounset=True)
assert isinstance(e.value, expandvars.UnboundVariable)
def run(spec_file):
if not os.path.exists(spec_file):
raise RunTimeError("Spec file not found: %s" % spec_file)
root_dir = os.path.dirname(spec_file)
with open(spec_file) as f:
spec_file_str = expandvars(f.read(), nounset=True)
global_spec = yaml.load(StringIO(spec_file_str),
Loader=yaml.FullLoader)
is_strict = global_spec.get("settings", {}).get("is_strict", True)
require_variables(global_spec.get("required_variables", []))
for output_filename, spec in global_spec["outputs"].items():
memo = dict(os.environ)
compile_files(root_dir, global_spec.get("global_dependencies", []),
is_strict)
compile_files(root_dir, spec.get("dependencies", []), is_strict)
try:
content = compile_files(root_dir, spec["targets"], is_strict)
finally:
os.environ.clear()
os.environ.update(memo)
with open(output_filename, 'w') as f:
f.write(content)
def test_invalid_operand_err():
os.environ = {"FOO": "damnbigfoobar"}
oprnds = "@#$%^&*()_'\"\\"
for o in oprnds:
with pytest.raises(ValueError) as e:
expandvars("${{FOO:{}}}".format(o))
assert str(e.value) == (
"{}: syntax error: operand expected (error token is {})").format(
o, repr(o))
with pytest.raises(ValueError) as e:
expandvars("${{FOO:0:{}}}".format(o))
assert str(e.value) == (
"{}: syntax error: operand expected (error token is {})").format(
o, repr(o))
def test_escape():
importlib.reload(expandvars)
assert expandvars.expandvars("\\$FOO\\$BAR") == "$FOO$BAR"
assert expandvars.expandvars("\\\\$FOO") == "\\foo"
assert expandvars.expandvars("$FOO\\$BAR") == "foo$BAR"
assert expandvars.expandvars("\\$FOO$BAR") == "$FOObar"
assert expandvars.expandvars("$FOO" "\\" "\\" "\\" "$BAR") == ("foo" "\\" "$BAR")
assert expandvars.expandvars("$FOO\\$") == "foo$"
assert expandvars.expandvars("$\\FOO") == "$\\FOO"
assert expandvars.expandvars("$\\$FOO") == "$$FOO"
assert expandvars.expandvars("\\$FOO") == "$FOO"
assert (
expandvars.expandvars("D:\\\\some\\windows\\path")
== "D:\\\\some\\windows\\path"
)
def test_expandvars_option_nounset_with_strict():
importlib.reload(expandvars)
with pytest.raises(expandvars.ExpandvarsException,
match="FOO: parameter null or not set") as e:
assert expandvars.expandvars("${FOO:?}", nounset=True)
assert isinstance(e.value, expandvars.ParameterNullOrNotSet)
def test_expandvars_substitute():
importlib.reload(expandvars)
assert expandvars.expandvars("${FOO:+foo}") == "foo"
assert expandvars.expandvars("${FOO+foo}") == "foo"
assert expandvars.expandvars("${BAR:+foo}") == ""
assert expandvars.expandvars("${BAR+foo}") == ""
assert expandvars.expandvars("${BAR:+}") == ""
assert expandvars.expandvars("${BAR+}") == ""
assert expandvars.expandvars("${BUZ:+foo}") == "foo"
assert expandvars.expandvars("${BUZ+foo}:bar") == "foo:bar"
def test_escape():
os.environ.update({"FOO": "foo", "BAR": "bar"})
assert expandvars("\\$FOO\\$BAR") == "$FOO$BAR"
assert expandvars("$FOO\\$BAR") == "foo$BAR"
assert expandvars("\\$FOO$BAR") == "$FOObar"
assert expandvars("$FOO" "\\" "\\" "\\" "$BAR") == ("foo" "\\" "\\" "$BAR")
assert expandvars("$FOO\\$") == "foo$"
assert expandvars("$\\FOO") == "$\\FOO"
assert expandvars("\\$FOO") == "$FOO"
assert expandvars("D:\\some\\windows\\path") == "D:\\some\\windows\\path"
def test_invalid_operand_err():
importlib.reload(expandvars)
oprnds = "@#$%^&*()_'\""
for o in oprnds:
with pytest.raises(expandvars.ExpandvarsException) as e:
expandvars.expandvars("${{FOO:0:{0}}}".format(o))
assert str(e.value) == ("FOO: operand expected (error token is {0})").format(
repr(o)
)
assert isinstance(e.value, expandvars.OperandExpected)
with pytest.raises(expandvars.ExpandvarsException) as e:
expandvars.expandvars("${{FOO:{0}:{0}}}".format(o))
assert str(e.value) == ("FOO: operand expected (error token is {0})").format(
repr(o)
)
assert isinstance(e.value, expandvars.OperandExpected)
def __init__(self,
default=None,
help=None,
dtype=None,
metavar='\b',
env=None,
nounset=False,
**kwargs):
"""
The proto object. The env attribute allows one to set the environment variable
from which this proto reads value from.
:param default:
:param help:
:param dtype:
:param metavar:
:param env: the environment variable for the default value -- in the next version could be set
automatically from the prefix of the class.
:param nonset: default to False, when true raises error for env var that are not set.
:param kwargs:
"""
from termcolor import colored
if default and not dtype:
dtype = type(default)
# only apply dtype to ENV, and when dtype is not None.
if env:
if nounset or env in os.environ:
default = os.environ[env]
elif "$" in env:
default = expandvars(env, nounset=nounset)
if dtype:
default = dtype(default)
default_str = str([default])[1:-1]
if len(default_str) > 45:
default_str = default_str[:42] + "..."
default_str = default_str.replace('%', '%%')
help_str = colored(
f"\t<{'any' if dtype is None else dtype.__name__}> ", "blue")
if env and env in os.environ:
help_str += colored("$" + env, 'magenta') + '='
if default_str:
help_str += colored(default_str, 'cyan') + " "
if help:
# todo: handle multi-line help strings. Parse and remove indent.
if len(help_str + help) > 60:
help_str += '\n' + help
else:
help_str += help
super().__init__(default=default,
help=help_str,
dtype=dtype,
metavar=metavar,
**kwargs)
def onecmd(self, line):
line = expandvars(line.strip())
cmd_args = shlex.split(line)
if len(cmd_args) == 0:
return
cmd = cmd_args[0]
self.lastcmd = line
if hasattr(self, 'do_' + cmd):
return getattr(self, 'do_' + cmd)(cmd_args[1:])
else:
return self.default(cmd_args, line)
