def new_func(*args, **kwargs):
"""Deprecation wrapper."""
invalid_args = []
named_args = tf_inspect.getcallargs(func, *args, **kwargs)
for arg_name, spec in iter(deprecated_positions.items()):
if (spec.position < len(args) and
not (spec.has_ok_value and
_same_value(named_args[arg_name], spec.ok_value))):
invalid_args.append(arg_name)
if is_varargs_deprecated and len(args) > len(arg_spec.args):
invalid_args.append(arg_spec.varargs)
if is_kwargs_deprecated and kwargs:
invalid_args.append(arg_spec.keywords)
for arg_name in deprecated_arg_names:
if (arg_name in kwargs and
not (deprecated_positions[arg_name].has_ok_value and
_same_value(named_args[arg_name],
deprecated_positions[arg_name].ok_value))):
invalid_args.append(arg_name)
for arg_name in invalid_args:
logging.warning(
'From %s: calling %s (from %s) with %s is deprecated and will '
'be removed %s.\nInstructions for updating:\n%s',
_call_location(), decorator_utils.get_qualified_name(func),
func.__module__, arg_name,
'in a future version' if date is None else ('after %s' % date),
instructions)
return func(*args, **kwargs)
def new_func(*args, **kwargs):
"""Deprecation wrapper."""
# TODO(apassos) figure out a way to have reasonable performance with
# deprecation warnings and eager mode.
if is_in_graph_mode.IS_IN_GRAPH_MODE() and _PRINT_DEPRECATION_WARNINGS:
invalid_args = []
named_args = tf_inspect.getcallargs(func, *args, **kwargs)
for arg_name, spec in iter(deprecated_positions.items()):
if (spec.position < len(args) and
not (spec.has_ok_value and
_same_value(named_args[arg_name], spec.ok_value))):
invalid_args.append(arg_name)
if is_varargs_deprecated and len(args) > len(arg_spec.args):
invalid_args.append(arg_spec.varargs)
if is_kwargs_deprecated and kwargs:
invalid_args.append(arg_spec.varkw)
for arg_name in deprecated_arg_names:
if (arg_name in kwargs and
not (deprecated_positions[arg_name].has_ok_value and
_same_value(named_args[arg_name],
deprecated_positions[arg_name].ok_value))):
invalid_args.append(arg_name)
for arg_name in invalid_args:
if (func, arg_name) not in _PRINTED_WARNING:
if warn_once:
_PRINTED_WARNING[(func, arg_name)] = True
logging.warning(
'From %s: calling %s (from %s) with %s is deprecated and will '
'be removed %s.\nInstructions for updating:\n%s',
_call_location(), decorator_utils.get_qualified_name(func),
func.__module__, arg_name,
'in a future version' if date is None else ('after %s' % date),
instructions)
return func(*args, **kwargs)
def _map_args(call_node, function):
"""Maps AST call nodes to the actual function's arguments.
Args:
call_node: ast.Call
function: Callable[..., Any], the actual function matching call_node
Returns:
Dict[Text, ast.AST], mapping each of the function's argument names to
the respective AST node.
Raises:
ValueError: if the default arguments are not correctly set
"""
args = call_node.args
kwds = {kwd.arg: kwd.value for kwd in call_node.keywords}
call_args = tf_inspect.getcallargs(function, *args, **kwds)
# Keyword arguments not specified in kwds will be mapped to their defaults,
# which are Python values. Since we don't currently have a way to transform
# those into AST references, we simply remove them. By convention, directives
# use UNSPECIFIED as default value for for optional arguments. No other
# defaults should be present.
unexpected_defaults = []
for k in call_args:
if (k not in kwds
and call_args[k] not in args
and call_args[k] is not directives.UNSPECIFIED):
unexpected_defaults.append(k)
if unexpected_defaults:
raise ValueError('Unexpected keyword argument values, %s, for function %s'
% (zip(unexpected_defaults,
[call_args[k] for k in unexpected_defaults]),
function))
return {k: v for k, v in call_args.items() if v is not directives.UNSPECIFIED}
def testBoundFuncWithOneParam(self):
class Test(object):
def bound(self):
pass
t = Test()
self.assertEqual({'self': t}, tf_inspect.getcallargs(t.bound))
def _map_args(call_node, function):
"""Maps AST call nodes to the actual function's arguments.
Args:
call_node: ast.Call
function: Callable[..., Any], the actual function matching call_node
Returns:
Dict[Text, ast.AST], mapping each of the function's argument names to
the respective AST node.
"""
args = call_node.args
kwds = {kwd.arg: kwd.value for kwd in call_node.keywords}
return tf_inspect.getcallargs(function, *args, **kwds)
def new_func(*args, **kwargs):
"""Deprecation wrapper."""
named_args = tf_inspect.getcallargs(func, *args, **kwargs)
for arg_name, arg_value in deprecated_kwargs.items():
if arg_name in named_args and named_args[arg_name] == arg_value:
logging.warning(
'From %s: calling %s (from %s) with %s=%s is deprecated and will '
'be removed %s.\nInstructions for updating:\n%s',
_call_location(), decorator_utils.get_qualified_name(func),
func.__module__, arg_name, arg_value,
'in a future version' if date is None else ('after %s' % date),
instructions)
return func(*args, **kwargs)
def testClassMethod(self):
class Test(object):
@classmethod
def test(cls, a, b=3, c='hello'):
return (a, b, c)
self.assertEqual({
'cls': Test,
'a': 5,
'b': 3,
'c': 'goodbye'
}, tf_inspect.getcallargs(Test.test, 5, c='goodbye'))
def testBoundFuncWithManyParamsAndDefaults(self):
class Test(object):
def bound(self, a, b=2, c='Hello'):
return (a, b, c)
t = Test()
self.assertEqual({
'self': t,
'a': 3,
'b': 2,
'c': 'Goodbye'
}, tf_inspect.getcallargs(t.bound, 3, c='Goodbye'))
def new_func(*args, **kwargs):
"""Deprecation wrapper."""
if _PRINT_DEPRECATION_WARNINGS:
named_args = tf_inspect.getcallargs(func, *args, **kwargs)
for arg_name, arg_value in deprecated_kwargs.items():
if arg_name in named_args and named_args[arg_name] == arg_value:
if (func, arg_name) not in _PRINTED_WARNING:
if warn_once:
_PRINTED_WARNING[(func, arg_name)] = True
logging.warning(
'From %s: calling %s (from %s) with %s=%s is deprecated and '
'will be removed %s.\nInstructions for updating:\n%s',
_call_location(), decorator_utils.get_qualified_name(func),
func.__module__, arg_name, arg_value, 'in a future version'
if date is None else ('after %s' % date), instructions)
return func(*args, **kwargs)
def testUsesOutermostDecoratorsArgSpec(self):
def func():
pass
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
decorated = tf_decorator.make_decorator(
func,
wrapper,
decorator_argspec=tf_inspect.ArgSpec(
args=['a', 'b', 'c'],
varargs=None,
keywords=None,
defaults=(3, 'hello')))
self.assertEqual({
'a': 4,
'b': 3,
'c': 'goodbye'
}, tf_inspect.getcallargs(decorated, 4, c='goodbye'))
def converted_call(f, recursive, verbose, arg_types, *args, **kwargs):
"""Compiles a function call inline."""
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
unknown_arg_value = object() # Sentinel for arguments of unknown value
if tf_inspect.isbuiltin(f):
return builtins.dynamic_builtin(f, *args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
effective_args = args
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
partial_types = (f_class, )
else:
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = (unknown_arg_value, ) + args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
for name, arg in arg_values.items():
if arg is unknown_arg_value:
continue
arg_class = arg.__class__
# If arg_value_hints specifies any name, use that instead.
if name not in arg_types:
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
converted_f = to_graph(target_entity,
recursive=recursive,
verbose=verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types)
return converted_f(*effective_args, **kwargs)
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if options.verbose:
logging.info('Converted call: {}; owner: {}'.format(f, owner))
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
unknown_arg_value = object() # Sentinel for arguments of unknown value
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner, ) + args
else:
effective_args = args
partial_types = (f_class, )
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
if arg is unknown_arg_value:
continue
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
converted_f = to_graph(target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators,
optional_features=options.optional_features)
return converted_f(*effective_args, **kwargs)
def testUnboundFuncWithTwoParamsDefaultOneKeywordSecond(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 1, 'b': 4}, tf_inspect.getcallargs(func, b=4))
def testUnboundFuncWithOneParamDefaultOneKeyword(self):
def func(a=1):
return a
self.assertEqual({'a': 3}, tf_inspect.getcallargs(func, a=3))
def testUnboundFuncWithOneParamDefaultOnePositional(self):
def func(a=0):
return a
self.assertEqual({'a': 1}, tf_inspect.getcallargs(func, 1))
def testUnboundFuncWithTwoParamsKeyword(self):
def func(a, b):
return (a, b)
self.assertEqual({'a': 6, 'b': 7}, tf_inspect.getcallargs(func, a=6, b=7))
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(
1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f,
owner, args, kwargs)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
if _is_known_loaded_type(f, 'weakref', 'ref'):
logging.log(2, 'Permanently whitelisted: %s: weakref', f)
return _call_unconverted(f, args, kwargs)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper')
or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
# Note: TF linter disallows importing inspect.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access
logging.log(2, 'Permanently whitelisted: %s: part of builtin module',
f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self, ) + args
else:
effective_args = args
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
effective_args = args
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
effective_args = (f, ) + args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=None,
arg_types=None,
experimental_optional_features=options.optional_features)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
callargs = tf_inspect.getcallargs(converted_f, *effective_args,
**kwargs)
formatted_callargs = '\n'.join(' {}: {}'.format(k, v)
for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f,
formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1,
'Error transforming entity %s',
target_entity,
exc_info=True)
if is_autograph_strict_conversion_mode():
raise
logging.warn(
'Entity %s could not be transformed and will be executed as-is.'
' Some features (e.g. tensor-dependent conditionals and loops) may not'
' work as expected.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the'
' AutoGraph team. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
result = converted_f(*effective_args, **kwargs)
return result
def testUnboundFuncWithOneParamPositional(self):
def func(a):
return a
self.assertEqual({'a': 5}, tf_inspect.getcallargs(func, 5))
def testReturnsEmptyWhenUnboundFuncHasNoParameters(self):
def empty():
pass
self.assertEqual({}, tf_inspect.getcallargs(empty))
def converted_call(f, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(1, 'Converted call: %s\n args: %s\n kwargs: %s\n', f,
args, kwargs)
if inspect_utils.isbuiltin(f):
if f is eval:
return py_builtins.eval_in_original_context(f, args, 1)
if kwargs:
return py_builtins.overload_of(f)(*args, **kwargs)
else:
return py_builtins.overload_of(f)(*args)
# TODO(mdan): Clean up the naming inconsistency.
if hasattr(f, 'autograph_info__') or hasattr(f, '__ag_compiled'):
logging.log(2, 'Permanently whitelisted: %s: already converted', f)
return _call_unconverted(f, args, kwargs)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper')
or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'):
logging.log(2, 'Permanently whitelisted: %s: lru_cache', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, inspect, re)):
logging.log(2, 'Permanently whitelisted: %s: part of builtin module',
f)
return _call_unconverted(f, args, kwargs)
# Custom ops and kernels are also permanently whitelisted.
# See tensorflow.framework.load_library.
if (hasattr(f, '__module__')
and hasattr(f.__module__, '_IS_TENSORFLOW_PLUGIN')):
logging.log(2, 'Permanently whitelisted: %s: TensorFlow plugin', f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
# TODO(b/120224672): This unwrapping should be done before the checks above.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
if kwargs is not None:
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self, ) + args
else:
effective_args = args
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
effective_args = (f, ) + args
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
effective_args = args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
if not tf_inspect.isclass(target_entity):
if not hasattr(target_entity, '__code__'):
logging.log(2, 'Permanently whitelisted: %s: native binding',
target_entity)
return _call_unconverted(f, args, kwargs)
elif (hasattr(target_entity.__code__, 'co_filename')
and target_entity.__code__.co_filename == '<string>'):
# TODO(mdan): __globals__['txt'] might work in Py3.
logging.log(
2, 'Permanently whitelisted: %s: dynamic code (exec?)',
target_entity)
return _call_unconverted(f, args, kwargs)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
experimental_optional_features=options.optional_features)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
if six.PY3:
logging.log(2, 'KW defaults of %s : %s', converted_f,
converted_f.__kwdefaults__)
if kwargs is not None:
callargs = tf_inspect.getcallargs(converted_f, *effective_args,
**kwargs)
else:
callargs = tf_inspect.getcallargs(converted_f, *effective_args)
formatted_callargs = '\n'.join(' {}: {}'.format(k, v)
for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f,
formatted_callargs)
except Exception as e: # pylint:disable=broad-except
logging.log(1,
'Error transforming entity %s',
target_entity,
exc_info=True)
if is_autograph_strict_conversion_mode():
raise
logging.warn(
'Entity %s could not be transformed and will be executed as-is.'
' Please report this to the AutoGraph team. When filing the bug, set'
' the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and'
' attach the full output. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
with StackTraceMapper(converted_f), tf_stack.CurrentModuleFilter():
try:
if kwargs is not None:
result = converted_f(*effective_args, **kwargs)
else:
result = converted_f(*effective_args)
except Exception as e:
_attach_metadata(e, converted_f, True)
raise
return result
def converted_call(f, args, kwargs, caller_fn_scope=None, options=None):
"""Compiles a function call inline.
For internal use only.
Note: The argument list is optimized for readability of generated code, which
may look like this:
ag__.converted_call(f, (arg1, arg2), None, fscope)
ag__.converted_call(f, (), dict(arg1=val1, **kwargs), fscope)
ag__.converted_call(f, (arg1, arg2) + varargs, dict(**kwargs), lscope)
Args:
f: The function to convert.
args: Tuple, the original positional arguments of f
kwargs: Optional[Dict], the original keyword arguments of f
caller_fn_scope: Optional[function_wrappers.FunctionScope], the function
scope of the converted function in which this call was originally made.
options: Optional[converter.ConversionOptions], conversion options. If not
specified, the value of caller_fn_scope.callopts is used. Either options
or caller_fn_scope must be present.
Returns:
Any, the result of executing a possibly-converted `f` with the given
arguments.
"""
logging.log(1, 'Converted call: %s\n args: %s\n kwargs: %s\n', f,
args, kwargs)
if options is None:
if caller_fn_scope is None:
raise ValueError(
'either caller_fn_scope or options must have a value')
options = caller_fn_scope.callopts
if conversion.is_in_whitelist_cache(f, options):
logging.log(2, 'Whitelisted %s: from cache')
return _call_unconverted(f, args, kwargs, options, False)
if ag_ctx.control_status_ctx().status == ag_ctx.Status.DISABLED:
logging.log(2, 'Whitelisted: %s: AutoGraph is disabled in context', f)
return _call_unconverted(f, args, kwargs, options, False)
if inspect_utils.isbuiltin(f):
if f is eval:
return py_builtins.eval_in_original_context(
f, args, caller_fn_scope)
if f is super:
return py_builtins.super_in_original_context(
f, args, caller_fn_scope)
if kwargs:
return py_builtins.overload_of(f)(*args, **kwargs)
else:
return py_builtins.overload_of(f)(*args)
if is_autograph_artifact(f):
logging.log(2, 'Permanently whitelisted: %s: AutoGraph artifact', f)
return _call_unconverted(f, args, kwargs, options)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper')
or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'{} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will run as-is.'
' You may still apply AutoGraph before the wrapt decorator.'.
format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs, options)
if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'):
logging.log(2, 'Permanently whitelisted: %s: lru_cache', f)
return _call_unconverted(f, args, kwargs, options)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs, options)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, inspect, re)):
logging.log(2, 'Permanently whitelisted: %s: part of builtin module',
f)
return _call_unconverted(f, args, kwargs, options)
# Custom ops and kernels are also permanently whitelisted.
# See tensorflow.framework.load_library.
if (hasattr(f, '__module__')
and hasattr(f.__module__, '_IS_TENSORFLOW_PLUGIN')):
logging.log(2, 'Permanently whitelisted: %s: TensorFlow plugin', f)
return _call_unconverted(f, args, kwargs, options)
if not options.user_requested and conversion.is_whitelisted(f):
return _call_unconverted(f, args, kwargs, options)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs, options)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
# TODO(b/120224672): This unwrapping should be done before the checks above.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
if kwargs is not None:
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self, ) + args
else:
effective_args = args
elif hasattr(f, '__class__') and hasattr(f.__class__, '__call__'):
# Callable objects. Dunder methods have special lookup rules, see:
# https://docs.python.org/3/reference/datamodel.html#specialnames
target_entity = f.__class__.__call__
effective_args = (f, ) + args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
if not tf_inspect.isclass(target_entity):
if not hasattr(target_entity, '__code__'):
logging.log(2, 'Permanently whitelisted: %s: native binding',
target_entity)
return _call_unconverted(f, args, kwargs, options)
elif (hasattr(target_entity.__code__, 'co_filename')
and target_entity.__code__.co_filename == '<string>'):
# TODO(mdan): __globals__['txt'] might work in Py3.
logging.log(
2, 'Permanently whitelisted: %s: dynamic code (exec?)',
target_entity)
return _call_unconverted(f, args, kwargs, options)
program_ctx = converter.ProgramContext(
options=options,
autograph_module=tf_inspect.getmodule(converted_call))
converted_f = conversion.convert(target_entity, program_ctx)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
if six.PY3:
logging.log(2, 'KW defaults of %s : %s', converted_f,
converted_f.__kwdefaults__)
if kwargs is not None:
callargs = tf_inspect.getcallargs(converted_f, *effective_args,
**kwargs)
else:
callargs = tf_inspect.getcallargs(converted_f, *effective_args)
formatted_callargs = '\n'.join(' {}: {}'.format(k, v)
for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f,
formatted_callargs)
except Exception as e: # pylint:disable=broad-except
logging.log(1,
'Error transforming entity %s',
target_entity,
exc_info=True)
if is_autograph_strict_conversion_mode():
raise
if isinstance(e, errors.UnsupportedLanguageElementError):
# Repeating the check made upon function entry because the state might
# have updated in the meantime.
if not conversion.is_in_whitelist_cache(f, options):
logging.warn(
'AutoGraph could not transform %s and will run it as-is.\n'
'Cause: %s', target_entity, e)
else:
logging.warn(
'AutoGraph could not transform %s and will run it as-is.\n'
'Please report this to the TensorFlow team. When filing the bug, set'
' the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and'
' attach the full output.\n'
'Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs, options)
with StackTraceMapper(converted_f), tf_stack.CurrentModuleFilter():
try:
if kwargs is not None:
result = converted_f(*effective_args, **kwargs)
else:
result = converted_f(*effective_args)
except Exception as e:
_attach_metadata(e, converted_f, True)
raise
return result
def testUnboundFuncWithTwoParamsDefaultOneKeywordSecond(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 1, 'b': 4}, tf_inspect.getcallargs(func, b=4))
def testUnboundFuncWithTwoParamsDefaultOneKeywordFirst(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 3, 'b': 2}, tf_inspect.getcallargs(func, a=3))
def testUnboundFuncWithOneParamDefaultOneKeyword(self):
def func(a=1):
return a
self.assertEqual({'a': 3}, tf_inspect.getcallargs(func, a=3))
def testUnboundFuncWithTwoParamsDefaultTwoKeywords(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 3, 'b': 4}, tf_inspect.getcallargs(func, a=3, b=4))
def converted_call(f, recursive, verbose, arg_types, *args, **kwargs):
"""Compiles a function call inline."""
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
unknown_arg_value = object() # Sentinel for arguments of unknown value
if inspect_utils.isbuiltin(f):
return builtins.dynamic_builtin(f, *args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
effective_args = args
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
partial_types = (f_class,)
else:
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
for name, arg in arg_values.items():
if arg is unknown_arg_value:
continue
arg_class = arg.__class__
# If arg_value_hints specifies any name, use that instead.
if name not in arg_types:
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
converted_f = to_graph(
target_entity,
recursive=recursive,
verbose=verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types)
return converted_f(*effective_args, **kwargs)
def testUnboundFuncWithOneParamKeyword(self):
def func(a):
return a
self.assertEqual({'a': 5}, tf_inspect.getcallargs(func, a=5))
def testUnboundFuncWithOneParamKeyword(self):
def func(a):
return a
self.assertEqual({'a': 5}, tf_inspect.getcallargs(func, a=5))
def testUnboundFuncWithOneParamDefault(self):
def func(a=13):
return a
self.assertEqual({'a': 13}, tf_inspect.getcallargs(func))
def testUnboundFuncWithOneParamDefault(self):
def func(a=13):
return a
self.assertEqual({'a': 13}, tf_inspect.getcallargs(func))
def testUnboundFuncWithOneParamDefaultOnePositional(self):
def func(a=0):
return a
self.assertEqual({'a': 1}, tf_inspect.getcallargs(func, 1))
def testUnboundFuncWithTwoParamsDefaultOnePositional(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 5, 'b': 2}, tf_inspect.getcallargs(func, 5))
def testUnboundFuncWithTwoParamsDefaultOnePositional(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 5, 'b': 2}, tf_inspect.getcallargs(func, 5))
def testUnboundFuncWithTwoParamsDefaultOneKeywordFirst(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 3, 'b': 2}, tf_inspect.getcallargs(func, a=3))
def __call__(self, inputs, *args, **kwargs):
"""A wrapper around `Network.call`.
A typical `call` method in a class subclassing `Network` will have a
signature that accepts `inputs`, as well as other `*args` and `**kwargs`.
`call` can optionally also accept `step_type` and `network_state`
(if `state_spec != ()` is not trivial). e.g.:
```python
def call(self,
inputs,
step_type=None,
network_state=(),
training=False):
...
return outputs, new_network_state
```
We will validate the first argument (`inputs`)
against `self.input_tensor_spec` if one is available.
If a `network_state` kwarg is given it is also validated against
`self.state_spec`. Similarly, the return value of the `call` method is
expected to be a tuple/list with 2 values: `(output, new_state)`.
We validate `new_state` against `self.state_spec`.
If no `network_state` kwarg is given (or if empty `network_state = ()` is
given, it is up to `call` to assume a proper "empty" state, and to
emit an appropriate `output_state`.
Args:
inputs: The input to `self.call`, matching `self.input_tensor_spec`.
*args: Additional arguments to `self.call`.
**kwargs: Additional keyword arguments to `self.call`.
These can include `network_state` and `step_type`. `step_type` is
required if the network's `call` requires it. `network_state` is
required if the underlying network's `call` requires it.
Returns:
A tuple `(outputs, new_network_state)`.
"""
if self.input_tensor_spec is not None:
nest_utils.assert_matching_dtypes_and_inner_shapes(
inputs,
self.input_tensor_spec,
allow_extra_fields=True,
caller=self,
tensors_name="`inputs`",
specs_name="`input_tensor_spec`")
call_argspec = tf_inspect.getargspec(self.call)
# Convert *args, **kwargs to a canonical kwarg representation.
normalized_kwargs = tf_inspect.getcallargs(
self.call, inputs, *args, **kwargs)
# TODO(b/156315434): Rename network_state to just state.
network_state = normalized_kwargs.get("network_state", None)
normalized_kwargs.pop("self", None)
if network_state not in (None, (), []):
nest_utils.assert_matching_dtypes_and_inner_shapes(
network_state,
self.state_spec,
allow_extra_fields=True,
caller=self,
tensors_name="`network_state`",
specs_name="`state_spec`")
if "step_type" not in call_argspec.args and not call_argspec.keywords:
normalized_kwargs.pop("step_type", None)
if (network_state in (None, ())
and "network_state" not in call_argspec.args
and not call_argspec.keywords):
normalized_kwargs.pop("network_state", None)
outputs, new_state = super(Network, self).__call__(**normalized_kwargs)
nest_utils.assert_matching_dtypes_and_inner_shapes(
new_state,
self.state_spec,
allow_extra_fields=True,
caller=self,
tensors_name="`new_state`",
specs_name="`state_spec`")
return outputs, new_state
def testUnboundFuncWithTwoParamsDefaultTwoKeywords(self):
def func(a=1, b=2):
return (a, b)
self.assertEqual({'a': 3, 'b': 4}, tf_inspect.getcallargs(func, a=3, b=4))
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if options.verbose >= converter.Verbosity.VERBOSE:
logging.info('Converted call: {}; owner: {}'.format(f, owner))
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
# Args typically include `self`, as required by the conversion process.
# When conversion is skipped, `self` is not necessary, because the
# original bound method is being executed. This code removes it.
if tf_inspect.ismethod(f) and args:
f_class = inspect_utils.getmethodclass(f)
if args[0] is f_class:
args = args[1:]
return f(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
# Unwrap functools.partial objects
# TODO(allenl, mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_class:
effective_args = (f_class,) + args
else:
effective_args = (f_class,) + args[1:]
partial_types = (f_class,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators,
optional_features=options.optional_features)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def testUnboundFuncWithTwoParamsPositional(self):
def func(a, b):
return (a, b)
self.assertEqual({'a': 10, 'b': 20}, tf_inspect.getcallargs(func, 10, 20))
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
unknown_arg_value = object() # Sentinel for arguments of unknown value
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
effective_args = args
partial_types = (f_class,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
if arg is unknown_arg_value:
continue
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators)
return converted_f(*effective_args, **kwargs)
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if options.verbose >= converter.Verbosity.VERBOSE:
logging.info('Converted call: {}; owner: {}'.format(f, owner))
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
# Args typically include `self`, as required by the conversion process.
# When conversion is skipped, `self` is not necessary, because the
# original bound method is being executed. This code removes it.
if tf_inspect.ismethod(f) and args:
f_class = inspect_utils.getmethodclass(f)
if args[0] is f_class:
args = args[1:]
return f(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
# TODO(mdan): This may be more elegantly handled using __get__?
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner, ) + args
else:
# Always override the self arg, because it might be different from
# what the method was bound to - see inspect_utils.getmethodclass.
assert args, 'Bound function call without self argument?'
effective_args = (f_class, ) + args[1:]
partial_types = (f_class, )
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
converted_f = to_graph(target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators,
optional_features=options.optional_features)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def testUnboundFuncWithTwoParamsKeyword(self):
def func(a, b):
return (a, b)
self.assertEqual({'a': 6, 'b': 7}, tf_inspect.getcallargs(func, a=6, b=7))
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
owner_attr = f
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if logging.has_verbosity(1):
if owner is not None:
composite_desc = '("{}" attr of {})'.format(owner_attr, owner)
else:
composite_desc = ''
logging.log(1,
'Converted call: %s %s\n args: %s\n kwargs: %s\n',
f, composite_desc, args, kwargs)
if inspect_utils.isbuiltin(f):
if kwargs:
return py_builtins.overload_of(f)(*args, **kwargs)
else:
return py_builtins.overload_of(f)(*args)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or
_is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'):
logging.log(2, 'Permanently whitelisted: %s: lru_cache', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if any(f in m.__dict__.values() for m in (collections, pdb, copy, inspect)):
logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
if kwargs is not None:
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self,) + args
else:
effective_args = args
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
effective_args = args
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
effective_args = (f,) + args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
if (not tf_inspect.isclass(target_entity) and
not hasattr(target_entity, '__code__')):
logging.log(
2, 'Permanently whitelisted: %s: native binding', target_entity)
return _call_unconverted(f, args, kwargs)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=None,
arg_types=None,
experimental_optional_features=options.optional_features)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
if kwargs is not None:
callargs = tf_inspect.getcallargs(
converted_f, *effective_args, **kwargs)
else:
callargs = tf_inspect.getcallargs(converted_f, *effective_args)
formatted_callargs = '\n'.join(
' {}: {}'.format(k, v) for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True)
if is_autograph_strict_conversion_mode():
raise
logging.warn(
'Entity %s could not be transformed and will be executed as-is.'
' Some features (e.g. tensor-dependent conditionals and loops) may not'
' work as expected.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the'
' AutoGraph team. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
if kwargs is not None:
result = converted_f(*effective_args, **kwargs)
else:
result = converted_f(*effective_args)
return result
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(1,
'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n',
f, owner, args, kwargs)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(b/122265385): Remove this bypass.
if ('wrapt' in sys.modules and
hasattr(sys.modules['wrapt'], 'FunctionWrapper') and
isinstance(f, sys.modules['wrapt'].FunctionWrapper)):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f), 1)
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return f(*args, **kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return f(*args, **kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
# Note: TF linter disallows importing inspect.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access
logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f)
return f(*args, **kwargs)
# TODO(mdan): This needs cleanup.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
# TODO(mdan): This may be inconsistent in certain situations.
# If the function had already been annotated with @tf.function, it
# may be bound to the incorrect object. It's unclear if those situations
# are possible, but if they happen, we need to check if f is bound
# to a shim like WeakrefSelf and unpack it.
# Args typically include `self`, as required by the conversion process.
# When conversion is skipped, `self` is not necessary, because the
# original bound method is being executed. This code removes it.
if tf_inspect.ismethod(f) and args:
f_self = inspect_utils.getmethodself(f)
if args[0] is f_self:
args = args[1:]
return f(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_self:
effective_args = (f_self,) + args
else:
effective_args = (f_self,) + args[1:]
partial_types = (f_self,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
raise NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
logging.log(3, 'Partial types in conversion of %s: %s', target_entity,
partial_types)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=arg_values,
arg_types=arg_types,
experimental_optional_features=options.optional_features,
experimental_strip_decorators=options.strip_decorators,
experimental_verbose=options.verbose,
experimental_partial_types=partial_types)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs)
formatted_callargs = '\n'.join(
' {}: {}'.format(k, v) for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True)
logging.warn(
'Entity %s could not be transformed and will be staged without change.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the'
' AutoGraph team. Cause: %s', target_entity, e)
return f(*args, **kwargs)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(1,
'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n',
f, owner, args, kwargs)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or
_is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
# Note: TF linter disallows importing inspect.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access
logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_self:
effective_args = (f_self,) + args
else:
effective_args = (f_self,) + args[1:]
partial_types = (f_self,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
raise NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
logging.log(3, 'Partial types in conversion of %s: %s', target_entity,
partial_types)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=arg_values,
arg_types=arg_types,
experimental_optional_features=options.optional_features,
experimental_strip_decorators=options.strip_decorators,
experimental_verbose=options.verbose,
experimental_partial_types=partial_types)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs)
formatted_callargs = '\n'.join(
' {}: {}'.format(k, v) for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True)
logging.warn(
'Entity %s could not be transformed and will be staged without change.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the'
' AutoGraph team. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def testReturnsEmptyWhenUnboundFuncHasNoParameters(self):
def empty():
pass
self.assertEqual({}, tf_inspect.getcallargs(empty))
def testUnboundFuncWithOneParamPositional(self):
def func(a):
return a
self.assertEqual({'a': 5}, tf_inspect.getcallargs(func, 5))
def testUnboundFuncWithTwoParamsPositional(self):
def func(a, b):
return (a, b)
self.assertEqual({'a': 10, 'b': 20}, tf_inspect.getcallargs(func, 10, 20))
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if options.verbose:
logging.info('Converted call: {}; owner: {}'.format(f, owner))
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner, ) + args
else:
effective_args = args
partial_types = (f_class, )
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
converted_f = to_graph(target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators,
optional_features=options.optional_features)
result = converted_f(*effective_args, **kwargs)
# When converting a function, we write a tmp file and import it as a module.
# This leaks the module's closure. Once we've executed the converted_f module
# and there is no more code left to be executed, we can clean up the module.
# TODO(mdan): Look into workarounds that don't suffer from refcount leaks.
# Possibly attach the closure as a regular closure cell, instead of relying on
# module globals.
# If there are callables in the result, they will fail to find their closure
# when called, so only delete module if all returned types are not callable.
flat_results = nest.flatten(result)
if all(map(_is_not_callable, flat_results)):
del sys.modules[converted_f.__module__]
return result
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(
1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f,
owner, args, kwargs)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if f in collections.__dict__.values():
return f(*args, **kwargs)
# TODO(mdan): This needs cleanup.
if (not options.force_conversion
and conversion.is_whitelisted_for_graph(f)):
# TODO(mdan): This may be inconsistent in certain situations.
# If the function had already been annotated with @tf.function, it
# may be bound to the incorrect object. It's unclear if those situations
# are possible, but if they happen, we need to check if f is bound
# to a shim like WeakrefSelf and unpack it.
# Args typically include `self`, as required by the conversion process.
# When conversion is skipped, `self` is not necessary, because the
# original bound method is being executed. This code removes it.
if tf_inspect.ismethod(f) and args:
f_self = inspect_utils.getmethodself(f)
if args[0] is f_self:
args = args[1:]
return f(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_self = inspect_utils.getmethodself(f)
if owner is not None:
partial_types = (type(owner), )
elif f_self is not None:
partial_types = (type(f_self), )
else:
partial_types = ()
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner, ) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_self:
effective_args = (f_self, ) + args
else:
effective_args = (f_self, ) + args[1:]
else:
effective_args = args
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
raise NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
logging.log(3, 'Partial types in conversion of %s: %s', target_entity,
partial_types)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=arg_values,
arg_types=arg_types,
experimental_optional_features=options.optional_features,
experimental_strip_decorators=options.strip_decorators,
experimental_verbose=options.verbose,
experimental_partial_types=partial_types)
if logging.has_verbosity(2):
callargs = tf_inspect.getcallargs(converted_f, *effective_args,
**kwargs)
formatted_callargs = '\n'.join(' {}: {}'.format(k, v)
for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f,
formatted_callargs)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
