def autodiff_tree(func, wrt, motion, mode, preserve_result, check_dims,
verbose):
"""Perform AD on all functions in a call tree.
This function walks the call tree and differentiates each function in it. It
also ensures that the global namespaces that each function in the call tree
was in are merged.
The `tangent` and `numpy` packages are added to the namespace here, so that
the gradient templates can assume that they are present.
Args:
See `grad`.
Returns:
final: A single module which contains the primals and adjoints of all the
functions in the call tree.
namespace: A merged dictionary with all the variables in the global
namespaces of each function. The primals and adjoints need access to
these in order to execute.
"""
# Imported here to avoid circular imports
import tangent
namespace = {'tangent': tangent, 'numpy': numpy}
done = set()
final = gast.Module(body=[])
namespace.update(six.get_function_globals(func))
node, required = autodiff_ast(func, wrt, motion, mode, preserve_result,
check_dims, verbose)
final.body.extend(node.body)
to_do = set(required)
if motion == 'split' and mode == 'reverse':
done.add((func, wrt))
to_do -= done
while to_do:
func, wrt = to_do.pop()
namespace.update(six.get_function_globals(func))
node, required = autodiff_ast(
func=func,
wrt=wrt,
motion='split',
mode=mode,
preserve_result=True,
check_dims=False,
verbose=verbose)
final.body.extend(node.body)
done.add((func, wrt))
to_do.update(required)
to_do -= done
return final, namespace
def _update_function(oldfunc, newfunc):
"""Update a function object."""
if _closure_changed(six.get_function_closure(oldfunc),
six.get_function_closure(newfunc)):
raise ClosureChanged()
setattr(oldfunc, six._func_code, six.get_function_code(newfunc))
setattr(oldfunc, six._func_defaults, six.get_function_defaults(newfunc))
_update_scope(six.get_function_globals(oldfunc),
six.get_function_globals(newfunc))
# XXX What else?
return oldfunc
def _update_function(oldfunc, newfunc):
"""Update a function object."""
if _closure_changed(six.get_function_closure(oldfunc),
six.get_function_closure(newfunc)):
raise ClosureChanged()
setattr(oldfunc, six._func_code, six.get_function_code(newfunc))
setattr(oldfunc, six._func_defaults, six.get_function_defaults(newfunc))
_update_scope(six.get_function_globals(oldfunc),
six.get_function_globals(newfunc))
# XXX What else?
return oldfunc
def _new_func_from_source(source, func):
"""
Create new function defined in source but maintain context from func
@param func: The function whose global + local context we will use
@param source: Python source code containing def statement
"""
src_str = ''.join(source)
frames = inspect.getouterframes(inspect.currentframe())
calling_frame = frames[2][0]
context = {}
# My initial instict was: exec src_str in func.func_globals.items(), calling_frame.f_locals
# however this seems to break the function closure so caveat here is that we create a new
# function with the locals merged into the globals.
#
# Possible consequences I can think of:
# - If a global exists that already has the same name as the local, it will be overwritten in
# in the context of this function. This shouldnt matter though as the global should have already
# been hidden by the new name?
#
# This functionality should be considered experimental as no idea what other consequences there
# could be.
#
# relevant: http://stackoverflow.com/questions/2749655/why-are-closures-broken-within-exec
globals = six.get_function_globals(func)
locals = calling_frame.f_locals
combined = globals.copy()
combined.update(locals)
Logger.debug('New src_str:\n %s' % src_str)
six.exec_(src_str, combined, context)
new_func = context[func.__name__]
return new_func
def function_to_graph(f, conversion_map, param_value_hints, owner_type=None):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
if owner_type is not None:
new_name = namer.compiled_function_name(f.__name__, f, owner_type)
else:
new_name = namer.compiled_function_name(f.__name__, f)
node.name = new_name
conversion_map.update_name_map(namer)
return node, conversion_map.name_map[f]
def to_graph(o, arg_value_hints=None):
"""Compile a Python entity into equivalent TensorFlow code.
Currently supported entities:
* functions
* classes
Classes are handled by converting all their methods into a new class.
Args:
o: A Python function or class.
arg_value_hints: A dict mapping parameter names to objects that can hint
at the type of those parameters.
Returns:
A function with a signature identical to `o`, but which when executed it
creates TF a graph that has the same functionality as the original entity.
"""
conversion_map = conversion.ConversionMap()
_, name = conversion.object_to_graph(o, conversion_map, arg_value_hints)
module = gast.Module([])
for import_line in config.COMPILED_IMPORT_STATEMENTS:
module.body.append(parser.parse_str(import_line))
for dep in conversion_map.dependency_cache.values():
module.body.append(dep)
compiled_node = compiler.ast_to_object(module)
# The compiled code should see everything the entry function saw.
# TODO(mdan): This might not work well if the call tree spans modules?
if tf_inspect.isfunction(o):
compiled_node.__dict__.update(six.get_function_globals(o))
compiled_fn = getattr(compiled_node, name)
return compiled_fn
def wrapper(*args, **kwargs):
func_name = func.__name__
caller_module_name = six.get_function_globals(func)['__name__']
logger.info("STARTED: {}.{}".format(
caller_module_name, func_name))
args_name = getargspec(func)[0]
args_dict = dict(zip(args_name, args))
for key, value in viewitems(args_dict):
if key == "self":
continue
logger.debug(" {} == {}".format(key, value))
if len(args) > len(args_name):
for i in range(len(args_name), len(args)):
logger.debug(" {}".format(args[i]))
for key, value in viewitems(kwargs):
logger.debug(" {} == {}".format(key, value))
start = time.time()
result = func(*args, **kwargs)
end = time.time()
logger.info(
"COMPLETED: {}.{} in {}s".format(
caller_module_name, func_name, round(end - start, 2)))
return result
def function_to_graph(f, conversion_map, param_value_hints, owner_type=None):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
if owner_type is not None:
new_name = namer.compiled_function_name(f.__name__, f, owner_type)
else:
new_name = namer.compiled_function_name(f.__name__, f)
node.name = new_name
conversion_map.update_name_map(namer)
return node, conversion_map.name_map[f]
def __init__(self, func, local, variables, param, dtype=None, **kwargs):
self.func = func
self.code = get_function_code(func)
self.dtype = 'float' if dtype == np.float32 else 'double'
self.local = local
self.param = param
self.variables = variables
self.float_char = 'f' if dtype == np.float32 else ''
self.func_globals = get_function_globals(self.func)
self.used_variables = set()
CodeGenerator.__init__(self, self.func, newline=';\n', offset=4, \
ostream=StringIO())
self.generate()
self.generate_signature()
fname = self.func.__name__
self.definition = template_device_func.render(
dtype = self.dtype,
name = fname,
local = self.local,
signature = self.signature,
used = self.used_variables,
src = self.ostream.getvalue()
)
self.invocation = "{}({});".format(fname, ", ".join(self.args))
def to_graph(f, arg_value_hints=None):
"""Compile a Python function into equivalent TensorFlow code.
Args:
f: A Python function with arbitrary arguments and return values.
arg_value_hints: A dict mapping parameter names to objects that can hint
at the type of those parameters.
Returns:
A function with a signature identical to `f`, but which when executed it
creates TF a graph that has the same functionality as the original function.
"""
conversion_map = conversion.ConversionMap()
_, name = conversion.object_to_graph(f, conversion_map, arg_value_hints)
module = gast.Module([])
for import_line in config.COMPILED_IMPORT_STATEMENTS:
module.body.append(parser.parse_str(import_line))
for dep in conversion_map.dependency_cache.values():
module.body.append(dep)
compiled_node = compiler.ast_to_object(module)
# The compiled code should see everything the entry function saw.
# TODO(mdan): This might not work well if the call tree spans modules?
compiled_node.__dict__.update(six.get_function_globals(f))
compiled_fn = getattr(compiled_node, name)
return compiled_fn
def to_graph(f, arg_value_hints=None):
"""Compile a Python function into equivalent TensorFlow code.
Args:
f: A Python function with arbitrary arguments and return values.
arg_value_hints: A dict mapping parameter names to objects that can hint
at the type of those parameters.
Returns:
A function with a signature identical to `f`, but which when executed it
creates TF a graph that has the same functionality as the original function.
"""
conversion_map = conversion.ConversionMap()
_, name = conversion.object_to_graph(f, conversion_map, arg_value_hints)
module = gast.Module([])
for import_line in config.COMPILED_IMPORT_STATEMENTS:
module.body.append(parser.parse_str(import_line))
for dep in conversion_map.dependency_cache.values():
module.body.append(dep)
compiled_node = compiler.ast_to_object(module)
# The compiled code should see everything the entry function saw.
# TODO(mdan): This might not work well if the call tree spans modules?
compiled_node.__dict__.update(six.get_function_globals(f))
compiled_fn = getattr(compiled_node, name)
return compiled_fn
def class_to_graph(c, conversion_map):
"""Specialization of `entity_to_graph` for classes."""
converted_members = {}
members = tf_inspect.getmembers(c, predicate=tf_inspect.ismethod)
if not members:
raise ValueError('Cannot convert %s: it has no member methods.')
class_globals = None
for _, m in members:
node, _ = function_to_graph(
m,
conversion_map=conversion_map,
arg_values={},
arg_types={'self': (c.__name__, c)},
owner_type=c)
# TODO(mdan): Do not assume all members have the same view of globals.
if class_globals is None:
class_globals = six.get_function_globals(m)
converted_members[m] = node
namer = conversion_map.new_namer(class_globals)
class_name = namer.compiled_class_name(c.__name__, c)
node = gast.ClassDef(
class_name,
bases=[],
keywords=[],
body=converted_members.values(),
decorator_list=[])
return node, class_name
def function_to_graph(f, conversion_map, arg_values, arg_types,
owner_type=None):
"""Specialization of `entity_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
namespace = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
namespace[fn.__name__] = fn
namer = conversion_map.new_namer(namespace)
ctx = context.EntityContext(
namer=namer,
source_code=tf_inspect.getsource(f),
source_file=tf_inspect.getfile(f),
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types)
node = node_to_graph(node, ctx, conversion_map.nocompile_decorators)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
if owner_type is not None:
new_name = namer.compiled_function_name(f.__name__, f, owner_type)
else:
new_name = namer.compiled_function_name(f.__name__, f)
node.name = new_name
conversion_map.update_name_map(namer)
return node, conversion_map.name_map[f]
def construct_new_test_function(original_func, name, build_params):
"""Builds a new test function based on parameterized data.
:param original_func: The original test function that is used as a template
:param name: The fullname of the new test function
:param build_params: A dictionary or list containing args or kwargs
for the new test
:return: A new function object
"""
new_func = types.FunctionType(
six.get_function_code(original_func),
six.get_function_globals(original_func),
name=name,
argdefs=six.get_function_defaults(original_func)
)
# Support either an arg list or kwarg dict for our data
build_args = build_params if isinstance(build_params, list) else []
build_kwargs = build_params if isinstance(build_params, dict) else {}
# Build a test wrapper to execute with our kwargs
def test_wrapper(func, test_args, test_kwargs):
@functools.wraps(func)
def wrapper(self):
return func(self, *test_args, **test_kwargs)
return wrapper
return test_wrapper(new_func, build_args, build_kwargs)
def _build_new_function(func, name):
code = six.get_function_code(func)
func_globals = six.get_function_globals(func)
func_defaults = six.get_function_defaults(func)
func_closure = six.get_function_closure(func)
return types.FunctionType(code, func_globals, name, func_defaults,
func_closure)
def parse_args(args, path, query, specials):
def one_or_many(fn_, dict_, key):
result = [fn_(value) for value in dict_[key]]
return result[0] if len(result) == 1 else result
kwargs = {}
for arg, parse_fn in six.iteritems(args):
if arg in specials:
kwargs[arg] = specials[arg]()
elif parse_fn is None:
kwargs[arg] = one_or_many(lambda x: x, query, arg)
elif isinstance(parse_fn, tuple):
kwargs[arg] = parse_fn[DEFAULT] if arg not in query else one_or_many(parse_fn[CALLABLE], query, arg)
elif isalambda(parse_fn):
_code = six.get_function_code(parse_fn)
closures = six.get_function_closure(parse_fn)
if closures:
assert len(closures) <= 1
fn = closures[0].cell_contents
else:
fn = eval(".".join(_code.co_names), six.get_function_globals(parse_fn))
kwargs[arg] = fn(**parse_args(get_function_args(parse_fn), path, query, specials))
else:
kwargs[arg] = one_or_many(parse_fn, query, arg)
return kwargs
def parse_args(args, path, query, specials):
def one_or_many(fn_, dict_, key):
result = [fn_(value) for value in dict_[key]]
return result[0] if len(result) == 1 else result
kwargs = {}
for arg, parse_fn in six.iteritems(args):
if arg in specials:
kwargs[arg] = specials[arg]()
elif parse_fn is None:
kwargs[arg] = one_or_many(lambda x: x, query, arg)
elif isinstance(parse_fn, tuple):
kwargs[
arg] = parse_fn[DEFAULT] if arg not in query else one_or_many(
parse_fn[CALLABLE], query, arg)
elif isalambda(parse_fn):
_code = six.get_function_code(parse_fn)
closures = six.get_function_closure(parse_fn)
if closures:
assert len(closures) <= 1
fn = closures[0].cell_contents
else:
fn = eval(".".join(_code.co_names),
six.get_function_globals(parse_fn))
kwargs[arg] = fn(**parse_args(get_function_args(parse_fn), path,
query, specials))
else:
kwargs[arg] = one_or_many(parse_fn, query, arg)
return kwargs
def __init__(self, model, **kwargs):
self.num = kwargs.pop('num', None)
ostream = StringIO()
code_gen = NumpyKernelGenerator(model,
model.ode,
offset=4,
ostream=ostream)
code_gen.generate()
post = model.__class__.post
for cls in model.__class__.__bases__:
if cls.__name__ == 'Model' and post != cls.post:
code_gen = NumpyKernelGenerator(model,
post,
offset=4,
ostream=ostream)
code_gen.generate()
break
self.source = ostream.getvalue()
self.func_globals = get_function_globals(model.ode)
self.name = "Numpy{}".format(model.__class__.__name__)
self.compile()
self.ode = MethodType(self.module.ode, model)
if 'post' in self.module.__dict__:
self.post = MethodType(self.module.post, model)
def function_to_graph(f, conversion_map, param_value_hints):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
namer.compiled_function_name(f.__name__, f)
conversion_map.add_to_cache(f, node)
conversion_map.update_name_map(namer)
# Recursively convert any remaining dependencies.
for obj in conversion_map.name_map.keys():
if obj not in conversion_map.dependency_cache:
object_to_graph(obj, conversion_map, None)
return node, conversion_map.name_map[f]
def class_to_graph(c, conversion_map, param_value_hints):
"""Specialization of `object_to_graph` for classes."""
converted_members = {}
members = tf_inspect.getmembers(c, predicate=tf_inspect.ismethod)
if not members:
raise ValueError('Cannot convert %s: it has no member methods.')
if 'self' in param_value_hints:
raise ValueError('Hints may not be provided for reserved name "self".')
param_value_hints['self'] = (c.__name__, c)
class_globals = None
for _, m in members:
node, _ = function_to_graph(m, conversion_map, param_value_hints, c)
# TODO(mdan): Do not assume all members have the same view of globals.
if class_globals is None:
class_globals = six.get_function_globals(m)
converted_members[m] = node
namer = conversion_map.new_namer(class_globals)
class_name = namer.compiled_class_name(c.__name__, c)
node = gast.ClassDef(
class_name,
bases=[],
keywords=[],
body=converted_members.values(),
decorator_list=[])
return node, class_name
def __init__(self, model, func, dtype, **kwargs):
self.dtype = dtype
self.model = model
self.func = func
self.float_char = 'f' if self.dtype == 'float' else ''
self.params_gdata = kwargs.pop('params_gdata', [])
self.inputs = kwargs.pop('inputs', dict())
self.variables = []
self.has_random = False
self.func_globals = get_function_globals(self.func)
CodeGenerator.__init__(self,
self.func,
newline=';\n',
offset=4,
ostream=StringIO(),
**kwargs)
_, self.signature, self.kwargs = self.extract_signature(self.func)
self.generate()
self.args = self.process_signature()
self.src = self.ostream.getvalue()
def _new_func_from_source(source, func):
"""
Create new function defined in source but maintain context from func
@param func: The function whose global + local context we will use
@param source: Python source code containing def statement
"""
src_str = ''.join(source)
frames = inspect.getouterframes(inspect.currentframe())
calling_frame = frames[2][0]
context = {}
# My initial instict was: exec src_str in func.func_globals.items(), calling_frame.f_locals
# however this seems to break the function closure so caveat here is that we create a new
# function with the locals merged into the globals.
#
# Possible consequences I can think of:
# - If a global exists that already has the same name as the local, it will be overwritten in
# in the context of this function. This shouldnt matter though as the global should have already
# been hidden by the new name?
#
# This functionality should be considered experimental as no idea what other consequences there
# could be.
#
# relevant: http://stackoverflow.com/questions/2749655/why-are-closures-broken-within-exec
globals = six.get_function_globals(func)
locals = calling_frame.f_locals
combined = globals.copy()
combined.update(locals)
Logger.debug('New src_str:\n %s' % src_str)
six.exec_(src_str, combined, context)
new_func = context[func.__name__]
return new_func
def getnamespace(f):
"""Returns the complete namespace of a function.
Namespace is defined here as the mapping of all non-local variables to values.
This includes the globals and the closure variables. Note that this captures
the entire globals collection of the function, and may contain extra symbols
that it does not actually use.
Args:
f: User defined function.
Returns:
A dict mapping symbol names to values.
"""
namespace = dict(six.get_function_globals(f))
closure = six.get_function_closure(f)
freevars = six.get_function_code(f).co_freevars
if freevars and closure:
for name, cell in zip(freevars, closure):
try:
namespace[name] = cell.cell_contents
except ValueError:
# Cell contains undefined variable, omit it from the namespace.
pass
return namespace
def class_to_graph(c, conversion_map, param_value_hints):
"""Specialization of `object_to_graph` for classes."""
converted_members = {}
members = tf_inspect.getmembers(c, predicate=tf_inspect.ismethod)
if not members:
raise ValueError('Cannot convert %s: it has no member methods.')
if 'self' in param_value_hints:
raise ValueError('Hints may not be provided for reserved name "self".')
param_value_hints['self'] = (c.__name__, c)
class_globals = None
for _, m in members:
node, _ = function_to_graph(m, conversion_map, param_value_hints, c)
# TODO(mdan): Do not assume all members have the same view of globals.
if class_globals is None:
class_globals = six.get_function_globals(m)
converted_members[m] = node
namer = conversion_map.new_namer(class_globals)
class_name = namer.compiled_class_name(c.__name__, c)
node = gast.ClassDef(
class_name,
bases=[],
keywords=[],
body=converted_members.values(),
decorator_list=[])
return node, class_name
def function_to_graph(f, conversion_map, param_value_hints):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
namer.compiled_function_name(f.__name__, f)
conversion_map.add_to_cache(f, node)
conversion_map.update_name_map(namer)
# Recursively convert any remaining dependencies.
for obj in conversion_map.name_map.keys():
if obj not in conversion_map.dependency_cache:
object_to_graph(obj, conversion_map, None)
return node, conversion_map.name_map[f]
def DataDrivenFixture(cls):
"""Generates new unittest test methods from methods defined in the
decorated class"""
if not issubclass(cls, TestCase):
raise DataDrivenFixtureError
test_case_attrs = dir(cls)
for attr_name in test_case_attrs:
if attr_name.startswith(DATA_DRIVEN_TEST_PREFIX) is False:
# Not a data driven test, skip it
continue
original_test = getattr(cls, attr_name, None).__func__
test_data = getattr(original_test, DATA_DRIVEN_TEST_ATTR, None)
if test_data is None:
# no data was provided to the datasource decorator or this is not a
# data driven test, skip it.
continue
for dataset in test_data:
# Name the new test based on original and dataset names
base_test_name = str(
original_test.__name__)[int(len(DATA_DRIVEN_TEST_PREFIX)):]
new_test_name = "test_{0}_{1}".format(base_test_name, dataset.name)
# Create a new test from the old test
new_test = FunctionType(six.get_function_code(original_test),
six.get_function_globals(original_test),
name=new_test_name)
# Copy over any other attributes the original test had (mainly to
# support test tag decorator)
for attr in list(set(dir(original_test)) - set(dir(new_test))):
setattr(new_test, attr, getattr(original_test, attr))
# Change the new test's default keyword values to the appropriate
# new data as defined by the datasource decorator
args, _, _, defaults = inspect.getargspec(original_test)
# Self doesn't have a default, so we need to remove it
args.remove('self')
# Make sure we take into account required arguments
kwargs = dict(
zip_longest(args[::-1],
list(defaults or ())[::-1],
fillvalue=None))
kwargs.update(dataset.data)
# Make sure the updated values are in the correct order
new_default_values = [kwargs[arg] for arg in args]
setattr(new_test, "func_defaults", tuple(new_default_values))
# Add the new test to the decorated TestCase
setattr(cls, new_test_name, new_test)
return cls
def _build_new_function(func, name):
code = six.get_function_code(func)
func_globals = six.get_function_globals(func)
func_defaults = six.get_function_defaults(func)
func_closure = six.get_function_closure(func)
return types.FunctionType(code, func_globals,
name, func_defaults,
func_closure)
def __init__(self, func):
self.func = func
self.namespace = six.get_function_globals(func)
if six.get_function_closure(func):
self.namespace.update(
dict(
zip(func.__code__.co_freevars,
(cell.cell_contents
for cell in six.get_function_closure(func)))))
def to_graph(e,
recursive=True,
verbose=False,
arg_values=None,
arg_types=None,
partial_types=None):
"""Compile a Python entity into equivalent TensorFlow code.
Currently supported entities:
* functions
* classes
Classes are handled by converting all their methods into a new class.
Args:
e: A Python entity.
recursive: Whether to recusrively convert any functions that the decorator
function may call.
verbose: Whether to output the compiled code in the logs.
arg_values: A dict containing value hints for symbols like function
parameters.
arg_types: A dict containing type hints for symbols like function
parameters.
partial_types: A set of types (e.g. classes) that will not be converted
entirely. Calls to member functions for these types will be renamed
independently.
Returns:
A function with a signature identical to `o`, but which when executed it
creates TF a graph that has the same functionality as the original entity.
"""
conversion_map = conversion.ConversionMap(
recursive=recursive,
nocompile_decorators=(convert, graph_ready, convert_inline),
partial_types=partial_types,
api_module=tf_inspect.getmodule(to_graph))
_, name = conversion.entity_to_graph(e, conversion_map, arg_values,
arg_types)
module = gast.Module([])
for import_line in config.COMPILED_IMPORT_STATEMENTS:
module.body.append(parser.parse_str(import_line))
for dep in conversion_map.dependency_cache.values():
module.body.append(dep)
compiled_node, compiled_src = compiler.ast_to_object(module)
# The compiled code should see everything the entry function saw.
# TODO(mdan): This might not work well if the call tree spans modules?
if tf_inspect.isfunction(e):
compiled_node.__dict__.update(six.get_function_globals(e))
compiled_fn = getattr(compiled_node, name)
if verbose:
logging.info('Compiled output of %s:\n\n%s\n', e, compiled_src)
return compiled_fn
def fix_js_args(func):
'''Use this function when unsure whether func takes this and arguments as its last 2 args.
It will append 2 args if it does not.'''
fcode = six.get_function_code(func)
fargs = fcode.co_varnames[fcode.co_argcount-2:fcode.co_argcount]
if fargs==('this', 'arguments') or fargs==('arguments', 'var'):
return func
code = append_arguments(six.get_function_code(func), ('this','arguments'))
return types.FunctionType(code, six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func))
def fix_js_args(func):
'''Use this function when unsure whether func takes this and arguments as its last 2 args.
It will append 2 args if it does not.'''
fcode = six.get_function_code(func)
fargs = fcode.co_varnames[fcode.co_argcount-2:fcode.co_argcount]
if fargs==('this', 'arguments') or fargs==('arguments', 'var'):
return func
code = append_arguments(six.get_function_code(func), ('this','arguments'))
return types.FunctionType(code, six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func))
def to_graph(e,
recursive=True,
verbose=False,
arg_values=None,
arg_types=None,
partial_types=None):
"""Compile a Python entity into equivalent TensorFlow code.
Currently supported entities:
* functions
* classes
Classes are handled by converting all their methods into a new class.
Args:
e: A Python entity.
recursive: Whether to recusrively convert any functions that the decorator
function may call.
verbose: Whether to output the compiled code in the logs.
arg_values: A dict containing value hints for symbols like function
parameters.
arg_types: A dict containing type hints for symbols like function
parameters.
partial_types: A set of types (e.g. classes) that will not be converted
entirely. Calls to member functions for these types will be renamed
independently.
Returns:
A function with a signature identical to `o`, but which when executed it
creates TF a graph that has the same functionality as the original entity.
"""
conversion_map = conversion.ConversionMap(
recursive=recursive,
nocompile_decorators=(convert, graph_ready, convert_inline),
partial_types=partial_types,
api_module=tf_inspect.getmodule(to_graph))
_, name = conversion.entity_to_graph(e, conversion_map, arg_values, arg_types)
module = gast.Module([])
for import_line in config.COMPILED_IMPORT_STATEMENTS:
module.body.append(parser.parse_str(import_line))
for dep in conversion_map.dependency_cache.values():
module.body.append(dep)
compiled_node, compiled_src = compiler.ast_to_object(module)
# The compiled code should see everything the entry function saw.
# TODO(mdan): This might not work well if the call tree spans modules?
if tf_inspect.isfunction(e):
compiled_node.__dict__.update(six.get_function_globals(e))
compiled_fn = getattr(compiled_node, name)
if verbose:
logging.info('Compiled output of %s:\n\n%s\n', e, compiled_src)
return compiled_fn
def DataDrivenFixture(cls):
"""Generates new unittest test methods from methods defined in the
decorated class"""
if not issubclass(cls, TestCase):
raise DataDrivenFixtureError
test_case_attrs = dir(cls)
for attr_name in test_case_attrs:
if attr_name.startswith(DATA_DRIVEN_TEST_PREFIX) is False:
# Not a data driven test, skip it
continue
original_test = getattr(cls, attr_name, None).__func__
test_data = getattr(original_test, DATA_DRIVEN_TEST_ATTR, None)
if test_data is None:
# no data was provided to the datasource decorator or this is not a
# data driven test, skip it.
continue
for dataset in test_data:
# Name the new test based on original and dataset names
base_test_name = str(original_test.__name__)[int(len(DATA_DRIVEN_TEST_PREFIX)) :]
new_test_name = "test_{0}_{1}".format(base_test_name, dataset.name)
# Create a new test from the old test
new_test = FunctionType(
six.get_function_code(original_test), six.get_function_globals(original_test), name=new_test_name
)
# Copy over any other attributes the original test had (mainly to
# support test tag decorator)
for attr in list(set(dir(original_test)) - set(dir(new_test))):
setattr(new_test, attr, getattr(original_test, attr))
# Change the new test's default keyword values to the appropriate
# new data as defined by the datasource decorator
args, _, _, defaults = inspect.getargspec(original_test)
# Self doesn't have a default, so we need to remove it
args.remove("self")
# Make sure we take into account required arguments
kwargs = dict(zip_longest(args[::-1], list(defaults or ())[::-1], fillvalue=None))
kwargs.update(dataset.data)
# Make sure the updated values are in the correct order
new_default_values = [kwargs[arg] for arg in args]
setattr(new_test, "func_defaults", tuple(new_default_values))
# Add the new test to the decorated TestCase
setattr(cls, new_test_name, new_test)
return cls
def execute(*args, **kwargs):
''' 修饰器代理函数 '''
start = time.time()
ret = func(*args, **kwargs)
logging.info("{module}.{func} execute sec:{total_sec}".format(
date=datetime.datetime.now().strftime("%m-%d %H:%M"),
module=six.get_function_globals(func).get('__name__', ''),
func=func.__name__,
total_sec=str(round(time.time() - start, 2))
))
return ret
def _environment(function, names_to_omit=()):
"""Yields the names and values visible from the function's scope."""
str_names_to_omit = set(map(str, names_to_omit))
for name, val in six.iteritems(six.get_function_globals(function)):
if str(name) not in str_names_to_omit:
yield name, val
closure = six.get_function_closure(function)
if closure is not None:
freevars = six.get_function_code(function).co_freevars
for name, cell in zip(freevars, closure):
if str(name) not in str_names_to_omit:
yield name, cell.cell_contents
def compile_func(func, variables, backend):
codegen = FuncGenerator(func, variables=variables, backend=backend)
src = codegen.generate()
co = compile(src, '<string>', 'exec')
locs = dict()
globals = dict.copy(get_function_globals(func))
eval(co, globals, locs)
ode = locs[func.__name__]
del locs
return ode, src
def test_getnamespace_hermetic(self):
# Intentionally hiding the global function to make sure we don't overwrite
# it in the global namespace.
free_function = object() # pylint:disable=redefined-outer-name
def test_fn():
return free_function
ns = inspect_utils.getnamespace(test_fn)
globs = six.get_function_globals(test_fn)
self.assertTrue(ns['free_function'] is free_function)
self.assertFalse(globs['free_function'] is free_function)
def test_getnamespace_hermetic(self):
# Intentionally hiding the global function to make sure we don't overwrite
# it in the global namespace.
free_function = object() # pylint:disable=redefined-outer-name
def test_fn():
return free_function
ns = inspect_utils.getnamespace(test_fn)
globs = six.get_function_globals(test_fn)
self.assertTrue(ns['free_function'] is free_function)
self.assertFalse(globs['free_function'] is free_function)
def decorator(caller, func=None):
"""
decorator(caller) converts a caller function into a decorator;
decorator(caller, func) decorates a function using a caller.
"""
if func is not None: # returns a decorated function
evaldict = six.get_function_globals(func).copy()
evaldict['_call_'] = caller
evaldict['_func_'] = func
return FunctionMaker.create(
func, "return _call_(_func_, %(shortsignature)s)",
evaldict, undecorated=func, __wrapped__=func)
else: # returns a decorator
if inspect.isclass(caller):
name = caller.__name__.lower()
callerfunc = get_init(caller)
doc = 'decorator(%s) converts functions/generators into ' \
'factories of %s objects' % (caller.__name__, caller.__name__)
fun = getfullargspec(callerfunc).args[1] # second arg
elif inspect.isfunction(caller):
name = '_lambda_' if caller.__name__ == '<lambda>' \
else caller.__name__
callerfunc = caller
doc = caller.__doc__
fun = getfullargspec(callerfunc).args[0] # first arg
else: # assume caller is an object with a __call__ method
name = caller.__class__.__name__.lower()
callerfunc = caller.__call__.im_func
doc = caller.__call__.__doc__
fun = getfullargspec(callerfunc).args[1] # second arg
evaldict = six.get_function_globals(callerfunc).copy()
evaldict['_call_'] = caller
evaldict['decorator'] = decorator
return FunctionMaker.create(
'%s(%s)' % (name, fun),
'return decorator(_call_, %s)' % fun,
evaldict, undecorated=caller, __wrapped__=caller,
doc=doc, module=caller.__module__)
def copy_func(f, name=None):
"""Create a copy of a function.
Parameters
----------
f : function
Function to copy.
name : str, optional
Name of new function.
"""
return types.FunctionType(six.get_function_code(f),
six.get_function_globals(f), name or f.__name__,
six.get_function_defaults(f), six.get_function_closure(f))
def execute(*args, **kwargs):
''' 修饰器代理函数 '''
try:
return func(*args, **kwargs)
except Exception:
arg_str = ",".join([get_arg_str(arg) for arg in args])
kwargs_str = ",".join(["%s=%s" % (k, get_arg_str(v)) for k, v in kwargs.items()])
func_args_str = ",".join([item for item in [arg_str, kwargs_str] if item])
logging.error('{module}.{func}({func_args_str}): {tb}'.format(
module=six.get_function_globals(func).get('__name__', ''),
func=func.__name__,
tb=traceback.format_exc(),
func_args_str=func_args_str,
))
def copy_func(f, name=None):
"""Create a copy of a function.
Parameters
----------
f : function
Function to copy.
name : str, optional
Name of new function.
"""
return types.FunctionType(six.get_function_code(f),
six.get_function_globals(f), name or f.__name__,
six.get_function_defaults(f), six.get_function_closure(f))
def __init__(self, func, fallback=None, autojit_kw=None):
self.func = func
# This covers a Python 2/3 change not covered by six
try:
self.func_name = func.__name__
except AttributeError:
self.func_name = func.func_name
module_name = inspect.getmoduleinfo(
six.get_function_globals(func)['__file__']).name
module_name = '.'.join(['trackpy', module_name])
self.module_name = module_name
self.autojit_kw = autojit_kw
if fallback is not None:
self.ordinary = fallback
else:
self.ordinary = func
def __init__(self, func, fallback=None, autojit_kw=None):
self.func = func
# This covers a Python 2/3 change not covered by six
try:
self.func_name = func.__name__
except AttributeError:
self.func_name = func.func_name
module_name = inspect.getmoduleinfo(
six.get_function_globals(func)['__file__']).name
module_name = '.'.join(['trackpy', module_name])
self.module_name = module_name
self.autojit_kw = autojit_kw
if fallback is not None:
self.ordinary = fallback
else:
self.ordinary = func
def function_to_graph(f, conversion_map, arg_values, arg_types,
owner_type=None):
"""Specialization of `entity_to_graph` for callable functions."""
node, source = parser.parse_entity(f)
node = node.body[0]
namespace = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
namespace[fn.__name__] = fn
# Manually add the utils namespace which may be used from generated code.
if 'py2tf_util' not in namespace:
namespace['py2tf_utils'] = utils
elif namespace['py2tf_utils'] != utils:
raise ValueError(
'The module name py2tf_utils is reserved and may not be used.')
namer = conversion_map.new_namer(namespace)
ctx = context.EntityContext(
namer=namer,
source_code=source,
source_file='<fragment>',
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types,
recursive=conversion_map.recursive)
node = node_to_graph(node, ctx, conversion_map.nocompile_decorators)
# TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py
new_name, did_rename = namer.compiled_function_name(f.__name__, f, owner_type)
if not did_rename:
new_name = f.__name__
if node.name != f.__name__:
raise NotImplementedError('Strange corner case. Send us offending code!')
node.name = new_name
conversion_map.update_name_map(namer)
return node, new_name
def getnamespace(f):
"""Returns the complete namespace of a function.
Namespace is defined here as the mapping of all non-local variables to values.
This includes the globals and the closure variables. Note that this captures
the entire globals collection of the function, and may contain extra symbols
that it does not actually use.
Args:
f: User defined function.
Returns:
A dict mapping symbol names to values.
"""
namespace = dict(six.get_function_globals(f))
closure = six.get_function_closure(f)
freevars = six.get_function_code(f).co_freevars
if freevars and closure:
for name, cell in zip(freevars, closure):
namespace[name] = cell.cell_contents
return namespace
def save_function(pickler, obj):
if not _locate_function(obj, pickler):
log.info("F1: %s" % obj)
globs = get_function_globals(obj)
mod_name = obj.__module__
pickler.save_reduce(_create_function, (get_function_code(obj),
{},
obj.__name__,
get_function_defaults(obj),
get_function_closure(obj),
obj.__dict__,
mod_name), obj=obj, func_globals=globs)
log.info("# F1 %s" % obj)
else:
log.info("F2: %s" % obj)
StockPickler.save_global(pickler, obj)
log.info("# F2 %s" % obj)
return
def test_get_function_globals():
def f():
pass
assert six.get_function_globals(f) is globals()
def translate(env, func, *args, **kwargs):
"""
Given a shellcode environment, a function and its parameters, translate
the function to a list of shellcode operations ready to be compiled or
assembled using :meth:`~pwnypack.shellcode.base.BaseEnvironment.compile`
or :meth:`~pwnypack.shellcode.base.BaseEnvironment.assemble`.
Arguments:
env(~pwnypack.shellcode.base.Base): An instance of a shellcode
environment.
func(callable): The function to translate to shellcode.
args(...): The positional arguments for the function.
kwargs(...): The keyword arguments for the function.
Returns:
list: The high-level shellcode operations.
"""
func_code = six.get_function_code(func)
func_globals = dict(__builtins__)
func_globals.update(six.get_function_globals(func))
ops = bc.disassemble(func_code.co_code)
program = []
f_args = inspect.getcallargs(func, *args, **kwargs)
variables = dict(
(func_code.co_varnames.index(arg_name), arg_value)
for arg_name, arg_value in six.iteritems(f_args)
)
stack = []
for op in ops:
if op.name == 'LOAD_CONST':
stack.append(func_code.co_consts[op.arg])
elif op.name == 'LOAD_GLOBAL':
global_name = func_code.co_names[op.arg]
stack.append(getattr(env, global_name, func_globals.get(global_name)))
elif op.name == 'LOAD_FAST':
var_name = func_code.co_varnames[op.arg]
stack.append(getattr(env, var_name, variables.get(op.arg)))
elif op.name == 'BUILD_LIST':
items = stack[-op.arg:]
del stack[-op.arg:]
stack.append(items)
elif op.name == 'LOAD_ATTR':
obj = stack.pop()
stack.append(getattr(obj, func_code.co_names[op.arg]))
elif op.name == 'CALL_FUNCTION':
nargs = op.arg & 0xff
nkwargs = op.arg >> 8
if nkwargs:
f_kwargs = dict(zip(stack[-nkwargs * 2::2], stack[-nkwargs * 2 + 1::2]))
del stack[-nkwargs * 2:]
else:
f_kwargs = {}
if nargs:
f_args = stack[-nargs:]
del stack[-nargs:]
else:
f_args = []
f = stack.pop()
if isinstance(f, Fragment):
stack.append(f(env, *f_args, **f_kwargs))
else:
stack.append(f(*f_args, **f_kwargs))
elif op.name == 'STORE_FAST':
value = stack.pop()
var_name = func_code.co_varnames[op.arg]
var = getattr(env, var_name, variables.get(op.arg, None))
if isinstance(var, Register):
program.append(LoadRegister(var, value))
else:
variables[op.arg] = value
elif op.name == 'POP_TOP':
value = stack.pop()
if isinstance(value, SyscallInvoke):
program.append(value)
elif isinstance(value, list):
program.extend(value)
else:
raise ValueError('No idea how to compile %s' % (value,))
elif op.name == 'RETURN_VALUE':
stack.pop()
elif op.name == 'DUP_TOP':
value = stack[-1]
if isinstance(value, SyscallInvoke):
stack.insert(-1, env.SYSCALL_RET_REG)
else:
stack.append(value)
elif op.name == 'BINARY_SUBSCR':
index = stack.pop()
value = stack.pop()
stack.append(value[index])
elif op.name == 'STORE_SUBSCR':
index = stack.pop()
value = stack.pop()
new_value = stack.pop()
var = value[index]
if isinstance(var, Register):
program.append(LoadRegister(var, new_value))
else:
value[index] = new_value
elif op.name == 'INPLACE_ADD':
value = stack.pop()
reg = stack.pop()
if not isinstance(reg, Register):
raise TypeError('In-place addition is only supported on registers')
program.extend(env.reg_add(reg, value))
stack.append(reg)
elif op.name == 'INPLACE_SUBTRACT':
value = stack.pop()
reg = stack.pop()
if not isinstance(reg, Register):
raise TypeError('In-place subtraction is only supported on registers')
program.extend(env.reg_sub(reg, value))
stack.append(reg)
else:
raise RuntimeError('Unsupported opcode: %s' % op.name)
return program
print(pos_to_inst[i])
print(pos_to_inst[i - 2])
print(list(map(chr, old_bytecode))[i - 4:i + 8])
print(bytelist[i - 4:i + 8])
break
raise RuntimeError(
'Your python version made changes to the bytecode')
check(six.get_function_code(check))
if __name__ == '__main__':
x = 'Wrong'
dick = 3000
def func(a):
print(x, y, z, a)
print(dick)
d = (x, )
for e in (e for e in x):
print(e)
return x, y, z
func2 = types.FunctionType(
append_arguments(six.get_function_code(func), ('x', 'y', 'z')),
six.get_function_globals(func),
func.__name__,
closure=six.get_function_closure(func))
args = (2, 2, 3, 4), 3, 4
assert func2(1, *args) == args
def get_function_module(func):
return get_function_globals(func).get('__name__')
from __future__ import absolute_import, print_function
import inspect
import logging
import traceback
import types
from tornado.httpserver import HTTPRequest
from six import iteritems, get_method_self, get_function_closure, get_function_code, get_function_globals
function_module = lambda func: get_function_globals(func).get('__name__')
function_closure_dict = lambda func: dict(zip(get_function_code(func).co_freevars,
(c.cell_contents for c in get_function_closure(func))))
class TornadoContextInspector(object):
"""
Tool for inspect and found HTTRequest from callback functions or stack frames.
This is useful for finding and logging the HTTPRequest object when an exception occur in async callback.
This class also can be used to generate traceback-like string for calls withing `tornado.gen` framework
Finding the actual request object in async callbacks is a pain, and this is can be done only by inspecting
function closures, objects that owning methods, and ...
The usage of `TornadoContextInspector` is simple
>>> import sys
>>> inspector = TornadoContextInspector()
>>> inspector.inspect_frame(sys.exc_info()[2].tb_frame)
>>> print(inspector.found_req) # may be None
HTTPRequest(protocol='http', ...)
>>> print(''.join(inspector.format_async_frames()))
File "file.py", line 32, in func_1
def _getparams_rtype(cls, function):
"""Get function params from input function and rtype.
:return: OrderedDict, rtype, vargs and kwargs.
:rtype: tuple
"""
try:
args, vargs, kwargs, default = getargspec(function)
except TypeError:
args, vargs, kwargs, default = (), (), (), ()
indexlen = len(args) - (0 if default is None else len(default))
params = OrderedDict()
for index, arg in enumerate(args):
pkwargs = {
'name': arg,
'mandatory': True
} # param kwargs
if index >= indexlen: # has default value
value = default[index - indexlen]
pkwargs['default'] = value
pkwargs['ref'] = None if value is None else data2schema(value)
pkwargs['mandatory'] = False
params[arg] = pkwargs
rtype = None
# parse docstring
if function.__doc__ is not None and not isbuiltin(function):
scope = get_function_globals(function)
for match in cls._REC.findall(function.__doc__):
if rtype is None:
rrtype = match[4].strip() or None
if rrtype:
rtypes = rrtype.split(',')
schemas = []
for rtype_ in rtypes:
rtype_ = rtype_.strip()
islist = False
try:
lkrtype = lookup(rtype_, scope=scope)
except ImportError:
islist = True
try:
if rtype_[-1] == 's':
lkrtype = lookup(
rtype_[:-1], scope=scope
)
elif rtype_.startswith('list of '):
lkrtype = lookup(
rtype_[8:], scope=scope
)
else:
raise
except ImportError:
msg = 'rtype "{0}" ({1}) from {2} not found.'
raise ImportError(
msg.format(rtype_, rrtype, function)
)
try:
schemacls = datatype2schemacls(lkrtype)
except TypeError:
schemacls = ParamTypeSchema(type=lkrtype)
rschema = schemacls()
if islist:
rschema = ArraySchema(itemtype=rschema)
schemas.append(rschema)
if len(rtypes) > 1:
rtype = OneOfSchema(schemas=schemas, nullable=True)
else:
rtype = schemas[0]
continue
pname = (match[1] or match[2]).strip()
if pname and pname in params:
ptype = (match[0] or match[3]).strip()
ptypes = ptype.split(',')
schemas = []
for ptype in ptypes:
ptype = ptype.strip()
islist = False
try:
lkptype = lookup(ptype, scope=scope)
except ImportError:
islist = True
try:
if ptype[-1] == 's':
lkptype = lookup(ptype[:-1], scope=scope)
elif ptype.startswith('list of '):
lkptype = lookup(ptype[8:], scope=scope)
else:
raise
except ImportError:
msg = 'Error on ptype "{0}" ({1}) from {2} not found.'
raise ImportError(
msg.format(pname, ptype, function)
)
try:
schemacls = datatype2schemacls(lkptype)
except TypeError:
schemacls = ParamTypeSchema(type=lkptype)
pschema = schemacls()
if islist:
pschema = ArraySchema(itemtype=pschema)
schemas.append(pschema)
if len(ptypes) > 1:
pschema = OneOfSchema(schemas=schemas, nullable=True)
else:
pschema = schemas[0]
params[pname]['ref'] = pschema
return params, rtype, vargs, kwargs
return [chr(op), chr(oparg & 255), chr((oparg >> 8) & 255)]
elif oparg <= 4294967296:
return [chr(opcode.EXTENDED_ARG),
chr((oparg >> 16) & 255),
chr((oparg >> 24) & 255),
chr(op),
chr(oparg & 255),
chr((oparg >> 8) & 255)]
else:
raise ValueError("Invalid oparg: {0} is too large".format(oparg))
if __name__=='__main__':
x = 'Wrong'
dick = 3000
def func(a):
print(x,y,z, a)
print(dick)
d = (x,)
for e in (e for e in x):
print(e)
return x, y, z
func2 =types.FunctionType(append_arguments(six.get_function_code(func), ('x', 'y', 'z')), six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func))
args = (2,2,3,4),3,4
assert func2(1, *args) == args
def pytest_runtest_setup(self, item):
# Check to see if we need to benchmark any invocations.
bench = item.keywords.get('bench')
if bench is None:
# Nope; nothing to see here.
# Check to see if we can skip this test (requested to /only/ run
# benchmarks).
if self.config.option.bench_only:
raise pytest.skip('no associated benchmark')
# Just continue to the test.
return
# Get the first argument to indicate what method to benchmark.
expression = bench.args[0]
iterations = bench.kwargs.get('iterations', 100)
# Create a wrapper for the test case that applies the benchmark.
item_function = self._item_function = item.function
item_function_globals = six.get_function_globals(item_function)
item_function_argspec = inspect.getargspec(item.function)
@wraps(item.function)
def item_function_wrapper(*args, **kwargs):
# Extract the function from the expression.
locals_, globals_ = locals(), item_function_globals
locals_.update(dict(zip(item_function_argspec.args, args)))
locals_.update(kwargs)
six.exec_('_function = %s' % expression, globals_, locals_)
_function = locals_['_function']
# Initialize benchmark process.
props = {'times': list()}
# Create a wrapper for the method to benchmark.
@wraps(_function)
def benchmark(*args, **kwargs):
# nonlocal elapsed, real_iterations
gc.collect()
gc.disable()
start = timer()
result = _function(*args, **kwargs)
finish = timer()
gc.enable()
props['times'].append(finish - start)
return result
# Replace the function with the wrapped function.
locals_['benchmark'] = benchmark
six.exec_('%s = benchmark' % expression, globals_, locals_)
# Attempt to replace it in global scope as well.
globals_.update(locals_)
# Get the (unbound) function.
try:
locals_['function'] = six.get_method_function(item_function)
except AttributeError:
locals_['function'] = item_function
# Iterate the set number of iterations.
item.teardown()
for _ in range(iterations):
item.setup()
locals_['args'] = args
locals_['kwargs'] = kwargs
six.exec_('function(*args, **kwargs)', globals_, locals_)
item.teardown()
# Restore the benchmarked function.
six.exec_('%s = _function' % expression, globals_, locals_)
# Construct a Benchmark instance to store the result.
self._benchmarks.append(Benchmark(item, **props))
if item.cls is not None:
setattr(item.cls, item.function.__name__, item_function_wrapper)
else:
item.obj = item_function_wrapper
def _compilecode(function, name, impl, args, varargs, kwargs):
"""Get generated code.
:return: function proxy generated code.
:rtype: str
"""
newcodestr, generatedname, impl_name = _generatecode(
function=function, name=name, impl=impl,
args=args, varargs=varargs, kwargs=kwargs
)
try:
__file__ = getfile(function)
except TypeError:
__file__ = '<string>'
# compile newcodestr
code = compile(newcodestr, __file__, 'single')
# define the code with the new function
_globals = {}
exec_(code, _globals)
# get new code
_var = _globals[generatedname]
newco = get_function_code(_var)
# get new consts list
newconsts = list(newco.co_consts)
if PY3:
newcode = list(newco.co_code)
else:
newcode = [ord(co) for co in newco.co_code]
consts_values = {impl_name: impl}
# change LOAD_GLOBAL to LOAD_CONST
index = 0
newcodelen = len(newcode)
while index < newcodelen:
if newcode[index] == LOAD_GLOBAL:
oparg = newcode[index + 1] + (newcode[index + 2] << 8)
name = newco.co_names[oparg]
if name in consts_values:
const_value = consts_values[name]
if const_value in newconsts:
pos = newconsts.index(const_value)
else:
pos = len(newconsts)
newconsts.append(consts_values[name])
newcode[index] = LOAD_CONST
newcode[index + 1] = pos & 0xFF
newcode[index + 2] = pos >> 8
index += 1
codeobj = getcodeobj(newconsts, newcode, newco, get_function_code(function))
# instanciate a new function
if function is None or isbuiltin(function):
result = FunctionType(codeobj, {})
else:
result = type(function)(
codeobj,
get_function_globals(function),
function.__name__,
get_function_defaults(function),
get_function_closure(function)
)
return result
