def test_restore_globals():
global_var_writer(10)
assert global_var_reader() == 10
assert global_var == 10
reader = Function.from_object(global_var_reader).eval()
writer = Function.from_object(global_var_writer).eval()
writer(20)
assert reader() == 20
assert global_var == 20
def check_component(component, func, additional_bindings=None,
expected_source=None, expected_new_bindings=None):
function = Function.from_object(func)
bindings = function.get_external_variables()
if additional_bindings is not None:
bindings.update(additional_bindings)
new_tree, new_bindings = component(function.tree, bindings)
if expected_source is None:
expected_ast = function.tree
else:
expected_ast = ast.parse(unindent(expected_source)).body[0]
assert_ast_equal(new_tree, expected_ast)
if expected_new_bindings is not None:
for k in expected_new_bindings:
if k not in new_bindings:
print('Expected binding missing:', k)
binding = new_bindings[k]
expected_binding = expected_new_bindings[k]
# Python 3.2 defines equality for range objects incorrectly
# (namely, the result is always False).
# So we just test it manually.
if sys.version_info < (3, 3) and isinstance(expected_binding, range):
assert type(binding) == type(expected_binding)
assert list(binding) == list(expected_binding)
else:
assert binding == expected_binding
def check_partial_apply(func, args=None, kwds=None,
expected_source=None, expected_new_bindings=None):
''' Test that with given constants, optimized_ast transforms
source to expected_source.
It :expected_new_bindings: is given, we check that they
are among new bindings returned by optimizer.
'''
if args is None:
args = tuple()
if kwds is None:
kwds = {}
new_func = partial_apply(func, *args, **kwds)
function = Function.from_object(new_func)
if expected_source is not None:
assert_ast_equal(function.tree, ast.parse(unindent(expected_source)).body[0])
if expected_new_bindings is not None:
for k in expected_new_bindings:
if k not in function.globals:
print('Expected binding missing:', k)
binding = function.globals[k]
expected_binding = expected_new_bindings[k]
# Python 3.2 defines equality for range objects incorrectly
# (namely, the result is always False).
# So we just test it manually.
if sys.version_info < (3, 3) and isinstance(expected_binding, range):
assert type(binding) == type(expected_binding)
assert list(binding) == list(expected_binding)
else:
assert binding == expected_binding
def partial_apply(func, *args, **kwds):
"""
Same as :func:`partial_eval`, but in addition uses the provided values of
positional and keyword arguments in the partial evaluation.
"""
function = Function.from_object(func, ignore_decorators=True)
if has_nested_definitions(function):
raise ValueError(
"A partially evaluated function cannot have nested function or class definitions")
if is_async(function):
raise ValueError("A partially evaluated function cannot be an async coroutine")
if len(args) > 0 or len(kwds) > 0:
bound_function = function.bind_partial(*args, **kwds)
else:
bound_function = function
new_tree, bindings = _run_components(
bound_function.tree,
bound_function.get_external_variables())
globals_ = dict(bound_function.globals)
globals_.update(bindings)
new_function = bound_function.replace(tree=new_tree, globals_=globals_)
return new_function.eval()
def check_partial_apply(func, args=None, kwds=None,
expected_source=None, expected_new_bindings=None):
"""
Test that with given constants, optimized_ast transforms
source to expected_source.
It :expected_new_bindings: is given, we check that they
are among new bindings returned by optimizer.
"""
if args is None:
args = tuple()
if kwds is None:
kwds = {}
new_func = partial_apply(func, *args, **kwds)
function = Function.from_object(new_func)
if expected_source is not None:
assert_ast_equal(function.tree, ast.parse(unindent(expected_source)).body[0])
if expected_new_bindings is not None:
for k in expected_new_bindings:
if k not in function.globals:
print('Expected binding missing:', k)
binding = function.globals[k]
expected_binding = expected_new_bindings[k]
assert binding == expected_binding
def check_component(component, func, additional_bindings=None,
expected_source=None, expected_new_bindings=None):
function = Function.from_object(func)
bindings = function.get_external_variables()
if additional_bindings is not None:
bindings.update(additional_bindings)
new_tree, new_bindings = component(function.tree, bindings)
if expected_source is None:
expected_ast = function.tree
else:
expected_ast = ast.parse(unindent(expected_source)).body[0]
assert_ast_equal(new_tree, expected_ast)
if expected_new_bindings is not None:
for k in expected_new_bindings:
if k not in new_bindings:
print('Expected binding missing:', k)
binding = new_bindings[k]
expected_binding = expected_new_bindings[k]
assert binding == expected_binding
def test_copy_globals():
"""
Checks that a restored function does not refer to the same globals dictionary,
but a copy of it. Therefore it cannot reassign global values.
"""
global_var_writer(10)
assert global_var_reader() == 10
assert global_var == 10
reader = Function.from_object(global_var_reader).eval()
writer = Function.from_object(global_var_writer).eval()
writer(20)
assert reader() == 10
assert global_var == 10
def function_from_source(source, globals_=None):
"""
A helper function to construct a Function object from a source
with custom __future__ imports.
"""
module = ast.parse(unindent(source))
ast.fix_missing_locations(module)
for stmt in module.body:
if type(stmt) == ast.FunctionDef:
tree = stmt
name = stmt.name
break
else:
raise ValueError("No function definitions found in the provided source")
code_object = compile(module, '<nofile>', 'exec', dont_inherit=True)
locals_ = {}
eval(code_object, globals_, locals_)
function_obj = locals_[name]
function_obj._peval_source = astunparse.unparse(tree)
return Function.from_object(function_obj)
def test_restore_simple_closure():
closure_ref = make_one_var_closure()
assert closure_ref() == 2
closure = Function.from_object(closure_ref).eval()
assert closure() == 3
assert closure_ref() == 4
def test_recursive_call():
# When evaluated inside a fake closure (to detect closure variables),
# the function name will be included in the list of closure variables
# (if it is used in the body of the function).
# So if the function was not a closure to begin with,
# the corresponding cell will be missing.
# This tests checks that Function evaluates non-closure functions
# without a fake closure to prevent that.
func = Function.from_object(recursive_outer)
func = func.replace()
func = func.eval()
assert func(10) == 1
func = Function.from_object(make_recursive())
func = func.replace()
func = func.eval()
assert func(10) == 2
def test_reapply_decorators():
@tag
def tagged(x):
return x
func = Function.from_object(tagged).eval()
assert '_tag' in vars(func) and vars(func)['_tag']
def test_bind_partial_varkwds():
func = Function.from_object(dummy_func_arg_groups)
new_func = func.bind_partial(1, 2, d=10).eval()
sig = inspect.signature(new_func)
assert new_func(3, 4) == (1, 2, (3, 4), {'d': 10})
assert 'a' not in sig.parameters
assert 'b' not in sig.parameters
assert 'args' in sig.parameters
assert 'kwds' not in sig.parameters
def test_bind_partial_args():
func = Function.from_object(dummy_func)
new_func = func.bind_partial(1).eval()
sig = funcsigs.signature(new_func)
assert new_func(2, 3, 4) == (1, 2, 3, 4)
assert 'a' not in sig.parameters
assert 'b' in sig.parameters
assert 'c' in sig.parameters
assert 'd' in sig.parameters
def test_bind_partial_kwds():
func = Function.from_object(dummy_func)
new_func = func.bind_partial(1, d=10).eval()
sig = inspect.signature(new_func)
assert new_func(2, 3) == (1, 2, 3, 10)
assert 'a' not in sig.parameters
assert 'b' in sig.parameters
assert 'c' in sig.parameters
assert 'd' not in sig.parameters
def test_get_source():
function = Function.from_object(sample_fn)
source = normalize_source(function.get_source())
expected_source = unindent(
"""
def sample_fn(x, y, foo='bar', **kw):
if (foo == 'bar'):
return (x + y)
else:
return kw['zzz']
""")
assert source == expected_source
def test_preserve_future_features():
# Test that the presence of a future feature is preserved after re-evaluation
src = """
from __future__ import division
def f():
return 1 / 2
"""
func = function_from_source(src)
new_func_obj = func.eval()
new_func = Function.from_object(new_func_obj)
assert new_func.future_features.division
assert new_func_obj() == 0.5
# Test that the absence of a future feature is preserved after re-evaluation
# (Does not test much in Py3, since all the future features are enabled by default).
src = """
def f():
return 1 / 2
"""
func = function_from_source(src)
new_func_obj = func.eval()
new_func = Function.from_object(new_func_obj)
if sys.version_info >= (3,):
assert new_func.future_features.division
assert new_func_obj() == 0.5
else:
assert not new_func.future_features.division
assert new_func_obj() == 0
def test_restore_modified_closure():
def remove_first_line(node):
assert isinstance(node, ast.FunctionDef)
node = copy.deepcopy(node)
node.body = node.body[1:]
return node
closure_ref = make_two_var_closure()
assert closure_ref() == 3
closure = Function.from_object(closure_ref)
closure = closure.replace(tree=remove_first_line(closure.tree)).eval()
assert closure() == 4
assert closure_ref() == 5
def test_preserve_future_feature_presence():
src = """
from __future__ import generator_stop
def f():
error = lambda: next(i for i in range(3) if i==10)
try:
return all(error() for i in range(2))
except RuntimeError:
return False
"""
func = function_from_source(src)
new_func_obj = func.eval()
new_func = Function.from_object(new_func_obj)
assert new_func.future_features.generator_stop
assert new_func_obj() == False
def _inline(node, gen_sym, return_name, constants):
"""
Return a list of nodes, representing inlined function call.
"""
fn = constants[node.func.id]
fn_ast = Function.from_object(fn).tree
gen_sym, new_fn_ast = mangle(gen_sym, fn_ast)
parameter_assignments = _build_parameter_assignments(node, new_fn_ast)
body_nodes = new_fn_ast.body
gen_sym, inlined_body, new_bindings = _wrap_in_loop(gen_sym, body_nodes, return_name)
constants = dict(constants)
constants.update(new_bindings)
return parameter_assignments + inlined_body, gen_sym, constants
def inline(func):
"""
Marks the function for inlining.
"""
function = Function.from_object(func, ignore_decorators=True)
if has_nested_definitions(function):
raise ValueError("An inlined function cannot have nested function or class definitions")
if is_a_generator(function):
raise ValueError("An inlined function cannot be a generator")
if is_async(function):
raise ValueError("An inlined function cannot be an async coroutine")
if len(function.closure_vals) > 0:
raise ValueError("An inlined function cannot have a closure")
func.__peval_inline__ = True
return func
def test_preserve_future_feature_absence():
src = """
def f():
error = lambda: next(i for i in range(3) if i==10)
try:
return all(error() for i in range(2))
except RuntimeError:
return False
"""
func = function_from_source(src)
new_func_obj = func.eval()
new_func = Function.from_object(new_func_obj)
if sys.version_info >= (3, 7):
assert new_func.future_features.generator_stop
assert new_func_obj() == False
else:
assert not new_func.future_features.generator_stop
assert new_func_obj() == True
def partial_apply(fn, *args, **kwds):
''' Return specialized version of fn, fixing given args and kwargs,
just as functools.partial does, but specialized function should be faster
'''
function = Function.from_object(fn)
if len(args) > 0 or len(kwds) > 0:
bound_function = function.bind_partial(*args, **kwds)
else:
bound_function = function
new_tree, bindings = optimized_ast(
bound_function.tree,
bound_function.get_external_variables())
globals_ = dict(bound_function.globals)
globals_.update(bindings)
new_function = bound_function.replace(tree=new_tree, globals_=globals_)
return new_function.eval()
def test_compile_ast():
function = Function.from_object(sample_fn)
compiled_fn = function.eval()
assert compiled_fn(3, -9) == sample_fn(3, -9)
assert compiled_fn(3, -9, 'z', zzz=map) == sample_fn(3, -9, 'z', zzz=map)
def test_construct_from_eval():
# Test that the function returned from Function.eval()
# can be used to construct a new Function object.
func = Function.from_object(dummy_func).eval()
func2 = Function.from_object(func).eval()
assert func2(1, 2, c=10) == (1, 2, 10, 5)
def test_closure_contents():
func = Function.from_object(make_one_var_closure())
assert 'global_var' not in func.closure_vals
assert 'closure_var' in func.closure_vals
