def test_container_to_text_default_encoding_and_err(test_input, expected):
"""
Test for passing objects to container_to_text(). Default encoding and errors
"""
assert container_to_text(test_input,
encoding=DEFAULT_ENCODING,
errors=DEFAULT_ERR_HANDLER) == expected
def test_container_to_text_incomp_encod_chars(test_input, encoding, errors,
expected):
"""
Test for passing incompatible characters and encodings container_to_text().
"""
assert container_to_text(test_input, encoding=encoding,
errors=errors) == expected
def assible_native_concat(nodes):
"""Return a native Python type from the list of compiled nodes. If the
result is a single node, its value is returned. Otherwise, the nodes are
concatenated as strings. If the result can be parsed with
:func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
string is returned.
https://github.com/pallets/jinja/blob/master/src/jinja2/nativetypes.py
"""
head = list(islice(nodes, 2))
if not head:
return None
if len(head) == 1:
out = _fail_on_undefined(head[0])
# TODO send unvaulted data to literal_eval?
if isinstance(out, AssibleVaultEncryptedUnicode):
return out.data
if isinstance(out, NativeJinjaText):
# Sometimes (e.g. ``| string``) we need to mark variables
# in a special way so that they remain strings and are not
# passed into literal_eval.
# See:
# https://github.com/assible/assible/issues/70831
# https://github.com/pallets/jinja/issues/1200
# https://github.com/assible/assible/issues/70831#issuecomment-664190894
return out
else:
if isinstance(nodes, types.GeneratorType):
nodes = chain(head, nodes)
out = u''.join([to_text(_fail_on_undefined(v)) for v in nodes])
try:
out = literal_eval(out)
if PY2:
# ensure bytes are not returned back into Assible from templating
out = container_to_text(out)
return out
except (ValueError, SyntaxError, MemoryError):
return out
def safe_eval(expr, locals=None, include_exceptions=False):
'''
This is intended for allowing things like:
with_items: a_list_variable
Where Jinja2 would return a string but we do not want to allow it to
call functions (outside of Jinja2, where the env is constrained).
Based on:
http://stackoverflow.com/questions/12523516/using-ast-and-whitelists-to-make-pythons-eval-safe
'''
locals = {} if locals is None else locals
# define certain JSON types
# eg. JSON booleans are unknown to python eval()
OUR_GLOBALS = {
'__builtins__': {}, # avoid global builtins as per eval docs
'false': False,
'null': None,
'true': True,
# also add back some builtins we do need
'True': True,
'False': False,
'None': None
}
# this is the whitelist of AST nodes we are going to
# allow in the evaluation. Any node type other than
# those listed here will raise an exception in our custom
# visitor class defined below.
SAFE_NODES = set((
ast.Add,
ast.BinOp,
# ast.Call,
ast.Compare,
ast.Dict,
ast.Div,
ast.Expression,
ast.List,
ast.Load,
ast.Mult,
ast.Num,
ast.Name,
ast.Str,
ast.Sub,
ast.USub,
ast.Tuple,
ast.UnaryOp,
))
# AST node types were expanded after 2.6
if sys.version_info[:2] >= (2, 7):
SAFE_NODES.update(set((ast.Set, )))
# And in Python 3.4 too
if sys.version_info[:2] >= (3, 4):
SAFE_NODES.update(set((ast.NameConstant, )))
# And in Python 3.6 too, although not encountered until Python 3.8, see https://bugs.python.org/issue32892
if sys.version_info[:2] >= (3, 6):
SAFE_NODES.update(set((ast.Constant, )))
filter_list = []
for filter_ in filter_loader.all():
filter_list.extend(filter_.filters().keys())
test_list = []
for test in test_loader.all():
test_list.extend(test.tests().keys())
CALL_WHITELIST = C.DEFAULT_CALLABLE_WHITELIST + filter_list + test_list
class CleansingNodeVisitor(ast.NodeVisitor):
def generic_visit(self, node, inside_call=False):
if type(node) not in SAFE_NODES:
raise Exception("invalid expression (%s)" % expr)
elif isinstance(node, ast.Call):
inside_call = True
elif isinstance(node, ast.Name) and inside_call:
# Disallow calls to builtin functions that we have not vetted
# as safe. Other functions are excluded by setting locals in
# the call to eval() later on
if hasattr(builtins,
node.id) and node.id not in CALL_WHITELIST:
raise Exception("invalid function: %s" % node.id)
# iterate over all child nodes
for child_node in ast.iter_child_nodes(node):
self.generic_visit(child_node, inside_call)
if not isinstance(expr, string_types):
# already templated to a datastructure, perhaps?
if include_exceptions:
return (expr, None)
return expr
cnv = CleansingNodeVisitor()
try:
parsed_tree = ast.parse(expr, mode='eval')
cnv.visit(parsed_tree)
compiled = compile(parsed_tree, to_native(expr), 'eval')
# Note: passing our own globals and locals here constrains what
# callables (and other identifiers) are recognized. this is in
# addition to the filtering of builtins done in CleansingNodeVisitor
result = eval(compiled, OUR_GLOBALS, dict(locals))
if PY2:
# On Python 2 u"{'key': 'value'}" is evaluated to {'key': 'value'},
# ensure it is converted to {u'key': u'value'}.
result = container_to_text(result)
if include_exceptions:
return (result, None)
else:
return result
except SyntaxError as e:
# special handling for syntax errors, we just return
# the expression string back as-is to support late evaluation
if include_exceptions:
return (expr, None)
return expr
except Exception as e:
if include_exceptions:
return (expr, e)
return expr
def test_container_to_text_different_types(test_input, expected, encoding,
errors):
"""Test for passing objects to container_to_text()."""
assert container_to_text(test_input, encoding=encoding,
errors=errors) == expected