def __init__(self, func):
self.func = func
self.namespace = six.get_function_globals(func)
if six.get_function_closure(func):
self.namespace.update(
dict(
zip(func.__code__.co_freevars,
(cell.cell_contents
for cell in six.get_function_closure(func)))))
def _update_function(oldfunc, newfunc):
"""Update a function object."""
if _closure_changed(six.get_function_closure(oldfunc),
six.get_function_closure(newfunc)):
raise ClosureChanged()
setattr(oldfunc, six._func_code, six.get_function_code(newfunc))
setattr(oldfunc, six._func_defaults, six.get_function_defaults(newfunc))
_update_scope(six.get_function_globals(oldfunc),
six.get_function_globals(newfunc))
# XXX What else?
return oldfunc
def _update_function(oldfunc, newfunc):
"""Update a function object."""
if _closure_changed(six.get_function_closure(oldfunc),
six.get_function_closure(newfunc)):
raise ClosureChanged()
setattr(oldfunc, six._func_code, six.get_function_code(newfunc))
setattr(oldfunc, six._func_defaults, six.get_function_defaults(newfunc))
_update_scope(six.get_function_globals(oldfunc),
six.get_function_globals(newfunc))
# XXX What else?
return oldfunc
def function_to_graph(f, conversion_map, param_value_hints):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
namer.compiled_function_name(f.__name__, f)
conversion_map.add_to_cache(f, node)
conversion_map.update_name_map(namer)
# Recursively convert any remaining dependencies.
for obj in conversion_map.name_map.keys():
if obj not in conversion_map.dependency_cache:
object_to_graph(obj, conversion_map, None)
return node, conversion_map.name_map[f]
def parse_args(args, path, query, specials):
def one_or_many(fn_, dict_, key):
result = [fn_(value) for value in dict_[key]]
return result[0] if len(result) == 1 else result
kwargs = {}
for arg, parse_fn in six.iteritems(args):
if arg in specials:
kwargs[arg] = specials[arg]()
elif parse_fn is None:
kwargs[arg] = one_or_many(lambda x: x, query, arg)
elif isinstance(parse_fn, tuple):
kwargs[
arg] = parse_fn[DEFAULT] if arg not in query else one_or_many(
parse_fn[CALLABLE], query, arg)
elif isalambda(parse_fn):
_code = six.get_function_code(parse_fn)
closures = six.get_function_closure(parse_fn)
if closures:
assert len(closures) <= 1
fn = closures[0].cell_contents
else:
fn = eval(".".join(_code.co_names),
six.get_function_globals(parse_fn))
kwargs[arg] = fn(**parse_args(get_function_args(parse_fn), path,
query, specials))
else:
kwargs[arg] = one_or_many(parse_fn, query, arg)
return kwargs
def getargspec(method):
"""
Drill through layers of decorators attempting to locate the actual argspec
for a method.
"""
argspec = inspect.getargspec(method)
args = argspec[0]
if args and args[0] == 'self':
return argspec
if hasattr(method, '__func__'):
method = method.__func__
func_closure = six.get_function_closure(method)
# NOTE(sileht): if the closure is None we cannot look deeper,
# so return actual argspec, this occurs when the method
# is static for example.
if func_closure is None:
return argspec
closure = next(
(
c for c in func_closure if six.callable(c.cell_contents)
),
None
)
method = closure.cell_contents
return getargspec(method)
def serialize(cust_obj):
"""A function to serialize custom objects passed to a model
Args:
cust_obj(callable): a custom layer or function to serialize
Returns:
a dict of the serialized components of the object"""
ser_func = dict()
if isinstance(cust_obj, types.FunctionType):
func_code = six.get_function_code(cust_obj)
func_code_d = dill.dumps(func_code).decode('raw_unicode_escape')
ser_func['func_code_d'] = func_code_d
ser_func['name_d'] = pickle.dumps(
cust_obj.__name__).decode('raw_unicode_escape')
ser_func['args_d'] = pickle.dumps(
six.get_function_defaults(cust_obj)).decode('raw_unicode_escape')
clos = dill.dumps(
six.get_function_closure(cust_obj)).decode('raw_unicode_escape')
ser_func['clos_d'] = clos
ser_func['type_obj'] = 'func'
else:
if hasattr(cust_obj, '__module__'): # pragma: no cover
cust_obj.__module__ = '__main__'
ser_func['name_d'] = None
ser_func['args_d'] = None
ser_func['clos_d'] = None
ser_func['type_obj'] = 'class'
loaded = dill.dumps(cust_obj).decode('raw_unicode_escape')
ser_func['func_code_d'] = loaded
return ser_func
def function_to_graph(f, conversion_map, param_value_hints, owner_type=None):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
if owner_type is not None:
new_name = namer.compiled_function_name(f.__name__, f, owner_type)
else:
new_name = namer.compiled_function_name(f.__name__, f)
node.name = new_name
conversion_map.update_name_map(namer)
return node, conversion_map.name_map[f]
def getrealname(method):
"""attempt to get a method's real name."""
argspec = inspect.getargspec(method)
args = argspec[0]
if args and args[0] == 'self':
return method.__name__
if hasattr(method, '__func__'):
method = method.__func__
func_closure = six.get_function_closure(method)
# NOTE(sileht): if the closure is None we cannot look deeper,
# so return actual argspec, this occurs when the method
# is static for example.
if func_closure is None:
return method.__name__
closure = next(
(
c for c in func_closure if six.callable(c.cell_contents)
),
None
)
method = closure.cell_contents
return getrealname(method)
def function_to_graph(f, conversion_map, param_value_hints):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
namer.compiled_function_name(f.__name__, f)
conversion_map.add_to_cache(f, node)
conversion_map.update_name_map(namer)
# Recursively convert any remaining dependencies.
for obj in conversion_map.name_map.keys():
if obj not in conversion_map.dependency_cache:
object_to_graph(obj, conversion_map, None)
return node, conversion_map.name_map[f]
def inspect_function_closures(self, func):
if self.found_req and self.stop_on_request_find:
return
if not get_function_closure(func):
if self.logger:
self.logger.debug('Function does not have any closure, skipping')
return
closures = function_closure_dict(func)
if func.__name__ == 'inner' and function_module(func) == 'tornado.gen':
# We are inside tornado.gen.Runner.run, continue to actual wrapped generator
generator_obj = closures['self'].gen
gen_frame = generator_obj.gi_frame
if gen_frame: # frame may be empty
if self.logger:
self.logger.debug('Found `tornado.gen` instance, running: %s, has_frame: %s'
% (generator_obj.gi_running, bool(gen_frame)))
# why? need test
# if not generator_obj.gi_running:
# # only write this line as async calls if generator is NOT running, if it's running it's present
# # on the normal traceback
self.async_frames.append(gen_frame)
if self.logger:
self.logger.debug('Diving into `tornado.gen` frame: %s' % traceback.format_stack(gen_frame, 1)[0])
self.inspect_dict(gen_frame.f_locals)
elif self.logger:
self.logger.debug('Found dead `tornado.gen` instance (without any frame), skipping')
return # it's a `tornado.gen` object, not need to dive into closures
if self.logger:
self.logger.debug('Cannot find generator, diving into closure variables')
return self.inspect_dict(closures)
def get_wrapped_function(function):
"""Get the method at the bottom of a stack of decorators."""
if not hasattr(function, six._func_closure) or \
not six.get_function_closure(function):
return function
def _get_wrapped_function(function):
if not hasattr(function, six._func_closure):
return None
func_closure = six.get_function_closure(function)
if not func_closure:
return None
for closure in func_closure:
func = closure.cell_contents
deeper_func = _get_wrapped_function(func)
if deeper_func:
return deeper_func
elif hasattr(closure.cell_contents, '__call__'):
return closure.cell_contents
return _get_wrapped_function(function)
def getrealname(method):
"""attempt to get a method's real name."""
argspec = inspect.getargspec(method)
args = argspec[0]
if args and args[0] == 'self':
return method.__name__
if hasattr(method, '__func__'):
method = method.__func__
func_closure = six.get_function_closure(method)
# NOTE(sileht): if the closure is None we cannot look deeper,
# so return actual argspec, this occurs when the method
# is static for example.
if func_closure is None:
return method.__name__
closure = next(
(
c for c in func_closure if six.callable(c.cell_contents)
),
None
)
method = closure.cell_contents
return getrealname(method)
def function_to_graph(f, conversion_map, param_value_hints, owner_type=None):
"""Specialization of `object_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
node_globals = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
node_globals[fn.__name__] = fn
namer = conversion_map.new_namer(node_globals)
node = node_to_graph(node, namer, node_globals, param_value_hints)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
if owner_type is not None:
new_name = namer.compiled_function_name(f.__name__, f, owner_type)
else:
new_name = namer.compiled_function_name(f.__name__, f)
node.name = new_name
conversion_map.update_name_map(namer)
return node, conversion_map.name_map[f]
def test_throttler(self):
threshold = 1
orig_function = mock.Mock()
# Add this magic name as it's required by functools
orig_function.__name__ = 'mock_func'
throttled_func = utils.throttler(threshold)(orig_function)
throttled_func()
sleep = utils.eventlet.sleep
def sleep_mock(amount_to_sleep):
sleep(amount_to_sleep)
self.assertGreater(threshold, amount_to_sleep)
with mock.patch.object(utils.eventlet, "sleep",
side_effect=sleep_mock):
throttled_func()
self.assertEqual(2, orig_function.call_count)
lock_with_timer = six.get_function_closure(
throttled_func)[1].cell_contents
timestamp = lock_with_timer.timestamp - threshold
lock_with_timer.timestamp = timestamp
throttled_func()
self.assertEqual(3, orig_function.call_count)
self.assertLess(timestamp, lock_with_timer.timestamp)
def _build_new_function(func, name):
code = six.get_function_code(func)
func_globals = six.get_function_globals(func)
func_defaults = six.get_function_defaults(func)
func_closure = six.get_function_closure(func)
return types.FunctionType(code, func_globals, name, func_defaults,
func_closure)
def run_while(cond_fn, body_fn, init_args):
"""Type-dependent functional while loop.
Args:
cond_fn: A Python callable implementing the stop conditions of the loop.
body_fn: A Python callable implementing the body of the loop.
init_args: The initial values of the arguments that will be passed to both
cond_fn and body_fn.
Returns:
result: A list of values with the same shape and type as init_args. If any
of the init_args, or any variables closed-over in cond_fn are Tensors,
tf.while_loop will be used, otherwise a Python while loop will be ran.
Raises:
ValueError: if init_args is not a tuple or list with one or more elements.
"""
if not isinstance(init_args, (tuple, list)) or not init_args:
raise ValueError(
'init_args must be a non-empty list or tuple, found %s' % init_args)
# TODO(alexbw): statically determine all active variables in cond_fn,
# and pass them directly
closure_vars = tuple(
[c.cell_contents for c in six.get_function_closure(cond_fn) or []])
possibly_tensors = tuple(init_args) + closure_vars
if is_tensor(*possibly_tensors):
return control_flow_ops.while_loop(cond_fn, body_fn, init_args)
else:
return py_while_loop(cond_fn, body_fn, init_args)
def serialize(cust_obj):
"""A function to serialize custom objects passed to a model
Args:
cust_obj(callable): a custom layer or function to serialize
Returns:
a dict of the serialized components of the object"""
ser_func = dict()
if isinstance(cust_obj, types.FunctionType):
func_code = six.get_function_code(cust_obj)
func_code_d = dill.dumps(func_code).decode('raw_unicode_escape')
ser_func['func_code_d'] = func_code_d
ser_func['name_d'] = pickle.dumps(
cust_obj.__name__).decode('raw_unicode_escape')
ser_func['args_d'] = pickle.dumps(
six.get_function_defaults(cust_obj)).decode('raw_unicode_escape')
clos = dill.dumps(
six.get_function_closure(cust_obj)).decode('raw_unicode_escape')
ser_func['clos_d'] = clos
ser_func['type_obj'] = 'func'
else:
if hasattr(cust_obj, '__module__'): # pragma: no cover
cust_obj.__module__ = '__main__'
ser_func['name_d'] = None
ser_func['args_d'] = None
ser_func['clos_d'] = None
ser_func['type_obj'] = 'class'
loaded = dill.dumps(cust_obj).decode('raw_unicode_escape')
ser_func['func_code_d'] = loaded
return ser_func
def construct_new_test_function(original_func, name, build_params):
"""Builds a new test function based on parameterized data.
:param original_func: The original test function that is used as a template
:param name: The fullname of the new test function
:param build_params: A dictionary or list containing args or kwargs
for the new test
:return: A new function object
"""
new_func = types.FunctionType(
six.get_function_code(original_func),
six.get_function_globals(original_func),
name=name,
argdefs=six.get_function_defaults(original_func),
closure=six.get_function_closure(original_func))
for key, val in original_func.__dict__.items():
if key != 'build_data':
new_func.__dict__[key] = val
# Support either an arg list or kwarg dict for our data
build_args = build_params if isinstance(build_params, list) else []
build_kwargs = build_params if isinstance(build_params, dict) else {}
# Build a test wrapper to execute with our kwargs
def test_wrapper(func, test_args, test_kwargs):
@functools.wraps(func)
def wrapper(self):
return func(self, *test_args, **test_kwargs)
return wrapper
return test_wrapper(new_func, build_args, build_kwargs)
def construct_new_test_function(original_func, name, build_params):
"""Builds a new test function based on parameterized data.
:param original_func: The original test function that is used as a template
:param name: The fullname of the new test function
:param build_params: A dictionary or list containing args or kwargs
for the new test
:return: A new function object
"""
new_func = types.FunctionType(
six.get_function_code(original_func),
six.get_function_globals(original_func),
name=name,
argdefs=six.get_function_defaults(original_func),
closure=six.get_function_closure(original_func)
)
for key, val in original_func.__dict__.items():
if key != 'build_data':
new_func.__dict__[key] = val
# Support either an arg list or kwarg dict for our data
build_args = build_params if isinstance(build_params, list) else []
build_kwargs = build_params if isinstance(build_params, dict) else {}
# Build a test wrapper to execute with our kwargs
def test_wrapper(func, test_args, test_kwargs):
@functools.wraps(func)
def wrapper(self):
return func(self, *test_args, **test_kwargs)
return wrapper
return test_wrapper(new_func, build_args, build_kwargs)
def run_while(cond_fn, body_fn, init_args):
"""Type-dependent functional while loop.
Args:
cond_fn: A Python callable implementing the stop conditions of the loop.
body_fn: A Python callable implementing the body of the loop.
init_args: The initial values of the arguments that will be passed to both
cond_fn and body_fn.
Returns:
result: A list of values with the same shape and type as init_args. If any
of the init_args, or any variables closed-over in cond_fn are Tensors,
tf.while_loop will be used, otherwise a Python while loop will be ran.
Raises:
ValueError: if init_args is not a tuple or list with one or more elements.
"""
if not isinstance(init_args, (tuple, list)) or not init_args:
raise ValueError(
'init_args must be a non-empty list or tuple, found %s' %
init_args)
# TODO(alexbw): statically determine all active variables in cond_fn,
# and pass them directly
closure_vars = tuple(
[c.cell_contents for c in six.get_function_closure(cond_fn) or []])
possibly_tensors = tuple(init_args) + closure_vars
if is_tensor(*possibly_tensors):
return control_flow_ops.while_loop(cond_fn, body_fn, init_args)
else:
return py_while_loop(cond_fn, body_fn, init_args)
def get_wrapped_function(function):
"""Get the method at the bottom of a stack of decorators."""
if not hasattr(function, six._func_closure) or \
not six.get_function_closure(function):
return function
def _get_wrapped_function(function):
if not hasattr(function, six._func_closure):
return None
func_closure = six.get_function_closure(function)
if not func_closure:
return None
for closure in func_closure:
func = closure.cell_contents
deeper_func = _get_wrapped_function(func)
if deeper_func:
return deeper_func
elif hasattr(closure.cell_contents, '__call__'):
return closure.cell_contents
return _get_wrapped_function(function)
def getnamespace(f):
"""Returns the complete namespace of a function.
Namespace is defined here as the mapping of all non-local variables to values.
This includes the globals and the closure variables. Note that this captures
the entire globals collection of the function, and may contain extra symbols
that it does not actually use.
Args:
f: User defined function.
Returns:
A dict mapping symbol names to values.
"""
namespace = dict(six.get_function_globals(f))
closure = six.get_function_closure(f)
freevars = six.get_function_code(f).co_freevars
if freevars and closure:
for name, cell in zip(freevars, closure):
try:
namespace[name] = cell.cell_contents
except ValueError:
# Cell contains undefined variable, omit it from the namespace.
pass
return namespace
def test_throttler(self):
threshold = 1
orig_function = mock.Mock()
# Add this magic name as it's required by functools
orig_function.__name__ = 'mock_func'
throttled_func = utils.throttler(threshold)(orig_function)
throttled_func()
sleep = utils.eventlet.sleep
def sleep_mock(amount_to_sleep):
sleep(amount_to_sleep)
self.assertGreater(threshold, amount_to_sleep)
with mock.patch.object(utils.eventlet, "sleep",
side_effect=sleep_mock):
throttled_func()
self.assertEqual(2, orig_function.call_count)
lock_with_timer = six.get_function_closure(
throttled_func)[1].cell_contents
timestamp = lock_with_timer.timestamp - threshold
lock_with_timer.timestamp = timestamp
throttled_func()
self.assertEqual(3, orig_function.call_count)
self.assertLess(timestamp, lock_with_timer.timestamp)
def function_to_graph(f, conversion_map, arg_values, arg_types,
owner_type=None):
"""Specialization of `entity_to_graph` for callable functions."""
node = parser.parse_object(f).body[0]
namespace = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
namespace[fn.__name__] = fn
namer = conversion_map.new_namer(namespace)
ctx = context.EntityContext(
namer=namer,
source_code=tf_inspect.getsource(f),
source_file=tf_inspect.getfile(f),
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types)
node = node_to_graph(node, ctx, conversion_map.nocompile_decorators)
# Simulate a rename to ensure the top level is in the name map. This is needed
# for top level functions, and it also helps the consistency verification made
# by update_name_map.
if owner_type is not None:
new_name = namer.compiled_function_name(f.__name__, f, owner_type)
else:
new_name = namer.compiled_function_name(f.__name__, f)
node.name = new_name
conversion_map.update_name_map(namer)
return node, conversion_map.name_map[f]
def parse_args(args, path, query, specials):
def one_or_many(fn_, dict_, key):
result = [fn_(value) for value in dict_[key]]
return result[0] if len(result) == 1 else result
kwargs = {}
for arg, parse_fn in six.iteritems(args):
if arg in specials:
kwargs[arg] = specials[arg]()
elif parse_fn is None:
kwargs[arg] = one_or_many(lambda x: x, query, arg)
elif isinstance(parse_fn, tuple):
kwargs[arg] = parse_fn[DEFAULT] if arg not in query else one_or_many(parse_fn[CALLABLE], query, arg)
elif isalambda(parse_fn):
_code = six.get_function_code(parse_fn)
closures = six.get_function_closure(parse_fn)
if closures:
assert len(closures) <= 1
fn = closures[0].cell_contents
else:
fn = eval(".".join(_code.co_names), six.get_function_globals(parse_fn))
kwargs[arg] = fn(**parse_args(get_function_args(parse_fn), path, query, specials))
else:
kwargs[arg] = one_or_many(parse_fn, query, arg)
return kwargs
def get_unwrapped_func(func):
"""Get original function under decorator.
Decorator hides original function inside itself. But in some cases it's
important to get access to original function, for ex: for documentation.
Args:
func (function): function that can be potentially a decorator which
hides original function
Returns:
function: unwrapped function or the same function
"""
if not inspect.isfunction(func) and not inspect.ismethod(func):
return func
if func.__name__ != six.get_function_code(func).co_name:
for cell in six.get_function_closure(func):
obj = cell.cell_contents
if inspect.isfunction(obj):
if func.__name__ == six.get_function_code(obj).co_name:
return obj
else:
return get_unwrapped_func(obj)
return func
def getargspec(method):
"""
Drill through layers of decorators attempting to locate the actual argspec
for a method.
"""
argspec = inspect.getargspec(method)
args = argspec[0]
if args and args[0] == 'self':
return argspec
if hasattr(method, '__func__'):
method = method.__func__
func_closure = six.get_function_closure(method)
# NOTE(sileht): if the closure is None we cannot look deeper,
# so return actual argspec, this occurs when the method
# is static for example.
if func_closure is None:
return argspec
closure = next((c for c in func_closure if six.callable(c.cell_contents)),
None)
method = closure.cell_contents
return getargspec(method)
def _build_new_function(func, name):
code = six.get_function_code(func)
func_globals = six.get_function_globals(func)
func_defaults = six.get_function_defaults(func)
func_closure = six.get_function_closure(func)
return types.FunctionType(code, func_globals,
name, func_defaults,
func_closure)
def test_get_function_closure():
def f():
x = 42
def g():
return x
return g
cell = six.get_function_closure(f())[0]
assert type(cell).__name__ == "cell"
def test_get_function_closure():
def f():
x = 42
def g():
return x
return g
cell = six.get_function_closure(f())[0]
assert type(cell).__name__ == "cell"
def _attach_closure(original_func, gen_func):
"""Attaches original_func's closure to gen_func.
Args:
original_func: original function
gen_func: generated function
Returns:
A new function with the complete closure
"""
closure = ()
gen_code = six.get_function_code(gen_func)
gen_closure = six.get_function_closure(gen_func)
if not gen_closure:
gen_closure = ()
original_code = six.get_function_code(original_func)
original_closure = six.get_function_closure(original_func)
if not original_closure:
original_closure = ()
gen_dict = {
free_var: cell
for free_var, cell in zip(gen_code.co_freevars, gen_closure)
}
original_dict = {
free_var: cell
for free_var, cell in zip(original_code.co_freevars, original_closure)
}
gen_dict.update(original_dict)
closure = tuple([gen_dict[cell] for cell in gen_code.co_freevars])
return types.FunctionType(
gen_code,
original_func.__globals__,
argdefs=original_func.__defaults__,
closure=closure)
def _resolve_methods(self):
import six
callback = self.pattern.callback
try:
closure = six.get_function_closure(callback)
code = six.get_function_code(callback)
while getattr(code, 'co_name') != 'view':
# lets unwrap!
view = getattr(closure[0], 'cell_contents')
closure = six.get_function_closure(view)
code = six.get_function_code(view)
freevars = code.co_freevars
except (AttributeError, IndexError):
raise RuntimeError('Unable to use callback invalid closure/function specified.')
else:
return closure[freevars.index('actions')].cell_contents
def _resolve_methods(self):
import six
callback = self.pattern.callback
try:
closure = six.get_function_closure(callback)
code = six.get_function_code(callback)
while getattr(code, 'co_name') != 'view':
# lets unwrap!
view = getattr(closure[0], 'cell_contents')
closure = six.get_function_closure(view)
code = six.get_function_code(view)
freevars = code.co_freevars
except (AttributeError, IndexError):
raise RuntimeError(
'Unable to use callback invalid closure/function specified.')
else:
return closure[freevars.index('actions')].cell_contents
def _environment(function, names_to_omit=()):
"""Yields the names and values visible from the function's scope."""
str_names_to_omit = set(map(str, names_to_omit))
for name, val in six.iteritems(six.get_function_globals(function)):
if str(name) not in str_names_to_omit:
yield name, val
closure = six.get_function_closure(function)
if closure is not None:
freevars = six.get_function_code(function).co_freevars
for name, cell in zip(freevars, closure):
if str(name) not in str_names_to_omit:
yield name, cell.cell_contents
def _resolve_methods(self):
from rest_framework_swagger.decorators import unwrap_decorator
callback = self.pattern.callback
closure = six.get_function_closure(callback)
code = six.get_function_code(callback)
if code and code.co_name == 'wrapped_view':
closure, code = unwrap_decorator(callback)
try:
freevars = code.co_freevars
except AttributeError:
raise RuntimeError('Unable to use callback invalid closure/function specified.')
return closure[freevars.index('actions')].cell_contents
def fix_js_args(func):
'''Use this function when unsure whether func takes this and arguments as its last 2 args.
It will append 2 args if it does not.'''
fcode = six.get_function_code(func)
fargs = fcode.co_varnames[fcode.co_argcount - 2:fcode.co_argcount]
if fargs == ('this', 'arguments') or fargs == ('arguments', 'var'):
return func
code = append_arguments(six.get_function_code(func), ('this', 'arguments'))
return types.FunctionType(code,
six.get_function_globals(func),
func.__name__,
closure=six.get_function_closure(func))
def __init__(self):
self._cache_groups = dict()
self._diff_running = False
regex = re.compile(r'^_cache_(.+)$')
for (_, m) in inspect.getmembers(
type(self),
predicate=lambda p:
(inspect.ismethod or inspect.isdatadescriptor)):
if hasattr(m, 'fget'):
f = m.fget
elif inspect.ismethod(m):
f = six.get_method_function(m)
elif inspect.isfunction(m):
f = m
else:
continue
fv = six.get_function_code(f).co_freevars
try:
closure = six.get_function_closure(f)
except AttributeError:
continue
if closure is None:
continue
vs = dict(zip(fv, (c.cell_contents for c in closure)))
# this is used to make sure we are in the right function
# i'm not proud of that, by the way
if '_cache_identifier_pj97YCjgnp' not in vs:
continue
try:
groups = vs['groups']
method_name = re.match(regex, vs['cache_var_name']).group(1)
except KeyError:
continue
for g in groups:
if g not in self._cache_groups:
self._cache_groups[g] = []
self._cache_groups[g].append(method_name)
setattr(self, '_cache_' + method_name, None)
setattr(self, '_cached_' + method_name, False)
setattr(self, '_cached_args_' + method_name, dict())
def __init__(self):
self._cache_groups = dict()
self._diff_running = False
regex = re.compile(r'^_cache_(.+)$')
for (_, m) in inspect.getmembers(type(self),
predicate=lambda p:
(inspect.ismethod or
inspect.isdatadescriptor)):
if hasattr(m, 'fget'):
f = m.fget
elif inspect.ismethod(m):
f = six.get_method_function(m)
elif inspect.isfunction(m):
f = m
else:
continue
fv = six.get_function_code(f).co_freevars
try:
closure = six.get_function_closure(f)
except AttributeError:
continue
if closure is None:
continue
vs = dict(zip(fv, (c.cell_contents for c in closure)))
# this is used to make sure we are in the right function
# i'm not proud of that, by the way
if '_cache_identifier_pj97YCjgnp' not in vs:
continue
try:
groups = vs['groups']
method_name = re.match(regex, vs['cache_var_name']).group(1)
except KeyError:
continue
for g in groups:
if g not in self._cache_groups:
self._cache_groups[g] = []
self._cache_groups[g].append(method_name)
setattr(self, '_cache_' + method_name, None)
setattr(self, '_cached_' + method_name, False)
setattr(self, '_cached_args_' + method_name, dict())
def copy_func(f, name=None):
"""Create a copy of a function.
Parameters
----------
f : function
Function to copy.
name : str, optional
Name of new function.
"""
return types.FunctionType(six.get_function_code(f),
six.get_function_globals(f), name or f.__name__,
six.get_function_defaults(f), six.get_function_closure(f))
def fix_js_args(func):
'''Use this function when unsure whether func takes this and arguments as its last 2 args.
It will append 2 args if it does not.'''
fcode = six.get_function_code(func)
fargs = fcode.co_varnames[fcode.co_argcount - 2:fcode.co_argcount]
if fargs == ('this', 'arguments') or fargs == ('arguments', 'var'):
return func
code = append_arguments(six.get_function_code(func), ('this', 'arguments'))
return types.FunctionType(
code,
six.get_function_globals(func),
func.__name__,
closure=six.get_function_closure(func))
def copy_func(f, name=None):
"""Create a copy of a function.
Parameters
----------
f : function
Function to copy.
name : str, optional
Name of new function.
"""
return types.FunctionType(six.get_function_code(f),
six.get_function_globals(f), name or f.__name__,
six.get_function_defaults(f), six.get_function_closure(f))
def get_decorators(function):
# If we have no func_closure, it means we are not wrapping any other functions.
decorators = []
try:
func_closure = six.get_function_closure(function)
except AttributeError:
return decorators
if not func_closure:
return [function]
# Otherwise, we want to collect all of the recursive results for every closure we have.
for closure in func_closure:
if isinstance(closure.cell_contents, types.FunctionType):
decorators.extend(get_decorators(closure.cell_contents))
return [function] + decorators
def _get_wrapped_function(function):
if not hasattr(function, six._func_closure):
return None
func_closure = six.get_function_closure(function)
if not func_closure:
return None
for closure in func_closure:
func = closure.cell_contents
deeper_func = _get_wrapped_function(func)
if deeper_func:
return deeper_func
elif hasattr(closure.cell_contents, '__call__'):
return closure.cell_contents
def _get_wrapped_function(function):
if not hasattr(function, six._func_closure):
return None
func_closure = six.get_function_closure(function)
if not func_closure:
return None
for closure in func_closure:
func = closure.cell_contents
deeper_func = _get_wrapped_function(func)
if deeper_func:
return deeper_func
elif hasattr(closure.cell_contents, '__call__'):
return closure.cell_contents
def function_to_graph(f,
conversion_map,
arg_values,
arg_types,
owner_type=None):
"""Specialization of `entity_to_graph` for callable functions."""
node, source = parser.parse_entity(f)
node = node.body[0]
namespace = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
namespace[fn.__name__] = fn
# Manually add the utils namespace which may be used from generated code.
if 'py2tf_util' not in namespace:
namespace['py2tf_utils'] = utils
elif namespace['py2tf_utils'] != utils:
raise ValueError(
'The module name py2tf_utils is reserved and may not be used.')
namer = conversion_map.new_namer(namespace)
ctx = context.EntityContext(namer=namer,
source_code=source,
source_file='<fragment>',
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types,
recursive=conversion_map.recursive)
node = node_to_graph(node, ctx, conversion_map.nocompile_decorators)
# TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py
new_name, did_rename = namer.compiled_function_name(
f.__name__, f, owner_type)
if not did_rename:
new_name = f.__name__
if node.name != f.__name__:
raise NotImplementedError(
'Strange corner case. Send us offending code!')
node.name = new_name
conversion_map.update_name_map(namer)
return node, new_name
def function_to_graph(f,
conversion_map,
arg_values,
arg_types,
owner_type=None):
"""Specialization of `entity_to_graph` for callable functions."""
node, source = parser.parse_entity(f)
node = node.body[0]
namespace = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
namespace[fn.__name__] = fn
_add_self_references(namespace, conversion_map.api_module)
namer = conversion_map.new_namer(namespace)
ctx = context.EntityContext(namer=namer,
source_code=source,
source_file='<fragment>',
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types,
owner_type=owner_type,
recursive=conversion_map.recursive)
node, deps = node_to_graph(node, ctx, conversion_map.nocompile_decorators)
# TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py
new_name, did_rename = namer.compiled_function_name(
f.__name__, f, owner_type)
if not did_rename:
new_name = f.__name__
if node.name != f.__name__:
raise NotImplementedError(
'Strange corner case. Send us offending code!')
node.name = new_name
conversion_map.update_name_map(namer)
# TODO(mdan): Use this at compilation.
conversion_map.additional_imports.update(deps)
return node, new_name
def attach(blueprint, decorator):
new_blueprint = copy.deepcopy(blueprint)
new_blueprint.deferred_functions = []
for func in blueprint.deferred_functions:
freevar_dict = dict(
zip(
six.get_function_code(func).co_freevars,
map(lambda f: f.cell_contents,
six.get_function_closure(func))))
rule = freevar_dict['rule']
endpoint = freevar_dict['endpoint']
view_func = freevar_dict['view_func']
options = freevar_dict['options']
decorated_view_func = decorator(view_func) if callable(view_func) and callable(decorator) \
else view_func
new_blueprint.add_url_rule(rule, endpoint, decorated_view_func,
**options)
return new_blueprint
def getargspec(method):
"""
Drill through layers of decorators attempting to locate the actual argspec
for a method.
"""
argspec = _getargspec(method)
args = argspec[0]
if args and args[0] == 'self':
return argspec
if hasattr(method, '__func__'):
method = method.__func__
func_closure = six.get_function_closure(method)
# NOTE(sileht): if the closure is None we cannot look deeper,
# so return actual argspec, this occurs when the method
# is static for example.
if not func_closure:
return argspec
closure = None
# In the case of deeply nested decorators (with arguments), it's possible
# that there are several callables in scope; Take a best guess and go
# with the one that looks most like a pecan controller function
# (has a __code__ object, and 'self' is the first argument)
func_closure = filter(
lambda c: (
six.callable(c.cell_contents) and
hasattr(c.cell_contents, '__code__')
),
func_closure
)
func_closure = sorted(
func_closure,
key=lambda c: 'self' in c.cell_contents.__code__.co_varnames,
reverse=True
)
closure = func_closure[0]
method = closure.cell_contents
return getargspec(method)
def function_to_graph(f, conversion_map, arg_values, arg_types,
owner_type=None):
"""Specialization of `entity_to_graph` for callable functions."""
node, source = parser.parse_entity(f)
node = node.body[0]
namespace = six.get_function_globals(f)
# This is needed for non-global functions.
closure = six.get_function_closure(f)
if closure:
for e in closure:
if callable(e.cell_contents):
fn = e.cell_contents
namespace[fn.__name__] = fn
# Manually add the utils namespace which may be used from generated code.
if 'py2tf_util' not in namespace:
namespace['py2tf_utils'] = utils
elif namespace['py2tf_utils'] != utils:
raise ValueError(
'The module name py2tf_utils is reserved and may not be used.')
namer = conversion_map.new_namer(namespace)
ctx = context.EntityContext(
namer=namer,
source_code=source,
source_file='<fragment>',
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types,
recursive=conversion_map.recursive)
node = node_to_graph(node, ctx, conversion_map.nocompile_decorators)
# TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py
new_name, did_rename = namer.compiled_function_name(f.__name__, f, owner_type)
if not did_rename:
new_name = f.__name__
if node.name != f.__name__:
raise NotImplementedError('Strange corner case. Send us offending code!')
node.name = new_name
conversion_map.update_name_map(namer)
return node, new_name
def save_function(pickler, obj):
if not _locate_function(obj, pickler):
log.info("F1: %s" % obj)
globs = get_function_globals(obj)
mod_name = obj.__module__
pickler.save_reduce(_create_function, (get_function_code(obj),
{},
obj.__name__,
get_function_defaults(obj),
get_function_closure(obj),
obj.__dict__,
mod_name), obj=obj, func_globals=globs)
log.info("# F1 %s" % obj)
else:
log.info("F2: %s" % obj)
StockPickler.save_global(pickler, obj)
log.info("# F2 %s" % obj)
return
def getnamespace(f):
"""Returns the complete namespace of a function.
Namespace is defined here as the mapping of all non-local variables to values.
This includes the globals and the closure variables. Note that this captures
the entire globals collection of the function, and may contain extra symbols
that it does not actually use.
Args:
f: User defined function.
Returns:
A dict mapping symbol names to values.
"""
namespace = dict(six.get_function_globals(f))
closure = six.get_function_closure(f)
freevars = six.get_function_code(f).co_freevars
if freevars and closure:
for name, cell in zip(freevars, closure):
namespace[name] = cell.cell_contents
return namespace
def getargspec(method):
"""
Drill through layers of decorators attempting to locate the actual argspec
for a method.
"""
argspec = _getargspec(method)
args = argspec[0]
if args and args[0] == 'self':
return argspec
if hasattr(method, '__func__'):
method = method.__func__
func_closure = six.get_function_closure(method)
# NOTE(sileht): if the closure is None we cannot look deeper,
# so return actual argspec, this occurs when the method
# is static for example.
if not func_closure:
return argspec
closure = None
# In the case of deeply nested decorators (with arguments), it's possible
# that there are several callables in scope; Take a best guess and go
# with the one that looks most like a pecan controller function
# (has a __code__ object, and 'self' is the first argument)
func_closure = filter(
lambda c: (six.callable(c.cell_contents) and hasattr(
c.cell_contents, '__code__')), func_closure)
func_closure = sorted(
func_closure,
key=lambda c: 'self' in c.cell_contents.__code__.co_varnames,
reverse=True)
closure = func_closure[0]
method = closure.cell_contents
return getargspec(method)
def _compilecode(function, name, impl, args, varargs, kwargs):
"""Get generated code.
:return: function proxy generated code.
:rtype: str
"""
newcodestr, generatedname, impl_name = _generatecode(
function=function, name=name, impl=impl,
args=args, varargs=varargs, kwargs=kwargs
)
try:
__file__ = getfile(function)
except TypeError:
__file__ = '<string>'
# compile newcodestr
code = compile(newcodestr, __file__, 'single')
# define the code with the new function
_globals = {}
exec_(code, _globals)
# get new code
_var = _globals[generatedname]
newco = get_function_code(_var)
# get new consts list
newconsts = list(newco.co_consts)
if PY3:
newcode = list(newco.co_code)
else:
newcode = [ord(co) for co in newco.co_code]
consts_values = {impl_name: impl}
# change LOAD_GLOBAL to LOAD_CONST
index = 0
newcodelen = len(newcode)
while index < newcodelen:
if newcode[index] == LOAD_GLOBAL:
oparg = newcode[index + 1] + (newcode[index + 2] << 8)
name = newco.co_names[oparg]
if name in consts_values:
const_value = consts_values[name]
if const_value in newconsts:
pos = newconsts.index(const_value)
else:
pos = len(newconsts)
newconsts.append(consts_values[name])
newcode[index] = LOAD_CONST
newcode[index + 1] = pos & 0xFF
newcode[index + 2] = pos >> 8
index += 1
codeobj = getcodeobj(newconsts, newcode, newco, get_function_code(function))
# instanciate a new function
if function is None or isbuiltin(function):
result = FunctionType(codeobj, {})
else:
result = type(function)(
codeobj,
get_function_globals(function),
function.__name__,
get_function_defaults(function),
get_function_closure(function)
)
return result
def _detach_methods(target, method_names):
for method_name in method_names:
if six.get_function_closure(getattr(target, method_name)):
orig_method = six.create_bound_method(
getattr(type(target), method_name), target)
setattr(target, method_name, orig_method)
def _detach_methods(target, method_names):
for method_name in method_names:
if six.get_function_closure(getattr(target, method_name)):
orig_method = six.create_bound_method(
getattr(type(target), method_name), target)
setattr(target, method_name, orig_method)
from __future__ import absolute_import, print_function
import inspect
import logging
import traceback
import types
from tornado.httpserver import HTTPRequest
from six import iteritems, get_method_self, get_function_closure, get_function_code, get_function_globals
function_module = lambda func: get_function_globals(func).get('__name__')
function_closure_dict = lambda func: dict(zip(get_function_code(func).co_freevars,
(c.cell_contents for c in get_function_closure(func))))
class TornadoContextInspector(object):
"""
Tool for inspect and found HTTRequest from callback functions or stack frames.
This is useful for finding and logging the HTTPRequest object when an exception occur in async callback.
This class also can be used to generate traceback-like string for calls withing `tornado.gen` framework
Finding the actual request object in async callbacks is a pain, and this is can be done only by inspecting
function closures, objects that owning methods, and ...
The usage of `TornadoContextInspector` is simple
>>> import sys
>>> inspector = TornadoContextInspector()
>>> inspector.inspect_frame(sys.exc_info()[2].tb_frame)
>>> print(inspector.found_req) # may be None
HTTPRequest(protocol='http', ...)
>>> print(''.join(inspector.format_async_frames()))
File "file.py", line 32, in func_1
def closure_n_code(func):
return unwrappage(
six.get_function_closure(func),
six.get_function_code(func))
return [chr(op), chr(oparg & 255), chr((oparg >> 8) & 255)]
elif oparg <= 4294967296:
return [chr(opcode.EXTENDED_ARG),
chr((oparg >> 16) & 255),
chr((oparg >> 24) & 255),
chr(op),
chr(oparg & 255),
chr((oparg >> 8) & 255)]
else:
raise ValueError("Invalid oparg: {0} is too large".format(oparg))
if __name__=='__main__':
x = 'Wrong'
dick = 3000
def func(a):
print(x,y,z, a)
print(dick)
d = (x,)
for e in (e for e in x):
print(e)
return x, y, z
func2 =types.FunctionType(append_arguments(six.get_function_code(func), ('x', 'y', 'z')), six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func))
args = (2,2,3,4),3,4
assert func2(1, *args) == args
