def test_is_whitelisted_for_graph(self):
def test_fn():
return constant_op.constant(1)
self.assertFalse(conversion.is_whitelisted_for_graph(test_fn))
self.assertTrue(conversion.is_whitelisted_for_graph(utils))
self.assertTrue(conversion.is_whitelisted_for_graph(constant_op.constant))
def test_is_whitelisted_for_graph(self):
def test_fn():
return constant_op.constant(1)
self.assertFalse(conversion.is_whitelisted_for_graph(test_fn))
self.assertTrue(conversion.is_whitelisted_for_graph(utils))
self.assertTrue(
conversion.is_whitelisted_for_graph(constant_op.constant))
def test_is_whitelisted_for_graph_tensorflow_like(self):
tf_like = imp.new_module('tensorflow_foo')
def test_fn():
pass
tf_like.test_fn = test_fn
test_fn.__module__ = tf_like
self.assertFalse(conversion.is_whitelisted_for_graph(tf_like.test_fn))
def test_is_whitelisted_for_graph_tensorflow_like(self):
tf_like = imp.new_module('tensorflow_foo')
def test_fn():
pass
tf_like.test_fn = test_fn
test_fn.__module__ = tf_like
self.assertFalse(conversion.is_whitelisted_for_graph(tf_like.test_fn))
def test_is_whitelisted_for_graph_callable_whitelisted_call(self):
whitelisted_mod = imp.new_module('test_whitelisted_call')
sys.modules['test_whitelisted_call'] = whitelisted_mod
config.CONVERSION_RULES = ((config.DoNotConvert('test_whitelisted_call'),) +
config.CONVERSION_RULES)
class TestClass(object):
def __call__(self):
pass
def whitelisted_method(self):
pass
TestClass.__module__ = 'test_whitelisted_call'
if six.PY2:
TestClass.__call__.__func__.__module__ = 'test_whitelisted_call'
else:
TestClass.__call__.__module__ = 'test_whitelisted_call'
class Subclass(TestClass):
def converted_method(self):
pass
tc = Subclass()
self.assertTrue(conversion.is_whitelisted_for_graph(TestClass.__call__))
self.assertTrue(conversion.is_whitelisted_for_graph(tc))
self.assertTrue(conversion.is_whitelisted_for_graph(tc.__call__))
self.assertTrue(conversion.is_whitelisted_for_graph(tc.whitelisted_method))
self.assertFalse(conversion.is_whitelisted_for_graph(Subclass))
self.assertFalse(conversion.is_whitelisted_for_graph(tc.converted_method))
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
logging.vlog(logging.DEBUG, 'Converted call: %s; owner: %s', f, owner)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
# Args typically include `self`, as required by the conversion process.
# When conversion is skipped, `self` is not necessary, because the
# original bound method is being executed. This code removes it.
if tf_inspect.ismethod(f) and args:
f_class = inspect_utils.getmethodclass(f)
if args[0] is f_class:
args = args[1:]
return f(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner, ) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_class:
effective_args = (f_class, ) + args
else:
effective_args = (f_class, ) + args[1:]
partial_types = (f_class, )
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=arg_values,
arg_types=arg_types,
experimental_optional_features=options.optional_features,
experimental_strip_decorators=options.strip_decorators,
experimental_verbose=options.verbose,
experimental_partial_types=partial_types)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(1,
'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n',
f, owner, args, kwargs)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or
_is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
# Note: TF linter disallows importing inspect.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access
logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_self:
effective_args = (f_self,) + args
else:
effective_args = (f_self,) + args[1:]
partial_types = (f_self,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
raise NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
logging.log(3, 'Partial types in conversion of %s: %s', target_entity,
partial_types)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=arg_values,
arg_types=arg_types,
experimental_optional_features=options.optional_features,
experimental_strip_decorators=options.strip_decorators,
experimental_verbose=options.verbose,
experimental_partial_types=partial_types)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs)
formatted_callargs = '\n'.join(
' {}: {}'.format(k, v) for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True)
logging.warn(
'Entity %s could not be transformed and will be staged without change.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the'
' AutoGraph team. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
unknown_arg_value = object() # Sentinel for arguments of unknown value
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
effective_args = args
partial_types = (f_class,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
if arg is unknown_arg_value:
continue
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators)
return converted_f(*effective_args, **kwargs)
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(1,
'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n',
f, owner, args, kwargs)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(b/122265385): Remove this bypass.
if ('wrapt' in sys.modules and
hasattr(sys.modules['wrapt'], 'FunctionWrapper') and
isinstance(f, sys.modules['wrapt'].FunctionWrapper)):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f), 1)
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return f(*args, **kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return f(*args, **kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if (f in collections.__dict__.values() or f in pdb.__dict__.values() or
f in copy.__dict__.values()):
logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f)
return f(*args, **kwargs)
# TODO(mdan): This needs cleanup.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
# TODO(mdan): This may be inconsistent in certain situations.
# If the function had already been annotated with @tf.function, it
# may be bound to the incorrect object. It's unclear if those situations
# are possible, but if they happen, we need to check if f is bound
# to a shim like WeakrefSelf and unpack it.
# Args typically include `self`, as required by the conversion process.
# When conversion is skipped, `self` is not necessary, because the
# original bound method is being executed. This code removes it.
if tf_inspect.ismethod(f) and args:
f_self = inspect_utils.getmethodself(f)
if args[0] is f_self:
args = args[1:]
return f(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_self:
effective_args = (f_self,) + args
else:
effective_args = (f_self,) + args[1:]
partial_types = (f_self,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
raise NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
logging.log(3, 'Partial types in conversion of %s: %s', target_entity,
partial_types)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=arg_values,
arg_types=arg_types,
experimental_optional_features=options.optional_features,
experimental_strip_decorators=options.strip_decorators,
experimental_verbose=options.verbose,
experimental_partial_types=partial_types)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs)
formatted_callargs = '\n'.join(
' {}: {}'.format(k, v) for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True)
logging.warn(
'Entity %s could not be transformed and will be staged without change.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY=5. Please report this to the AutoGraph'
' team. Cause: %s', target_entity, e)
return f(*args, **kwargs)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
logging.log(
1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f,
owner, args, kwargs)
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
if kwargs:
return py_builtins.overload_of(f)(*args, **kwargs)
else:
return py_builtins.overload_of(f)(*args)
if _is_known_loaded_type(f, 'weakref', 'ref'):
logging.log(2, 'Permanently whitelisted: %s: weakref', f)
return _call_unconverted(f, args, kwargs)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper')
or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
# Note: TF linter disallows importing inspect.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access
logging.log(2, 'Permanently whitelisted: %s: part of builtin module',
f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
if kwargs is not None:
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self, ) + args
else:
effective_args = args
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
effective_args = args
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
effective_args = (f, ) + args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=None,
arg_types=None,
experimental_optional_features=options.optional_features)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
if kwargs is not None:
callargs = tf_inspect.getcallargs(converted_f, *effective_args,
**kwargs)
else:
callargs = tf_inspect.getcallargs(converted_f, *effective_args)
formatted_callargs = '\n'.join(' {}: {}'.format(k, v)
for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f,
formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1,
'Error transforming entity %s',
target_entity,
exc_info=True)
if is_autograph_strict_conversion_mode():
raise
logging.warn(
'Entity %s could not be transformed and will be executed as-is.'
' Some features (e.g. tensor-dependent conditionals and loops) may not'
' work as expected.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the'
' AutoGraph team. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
if kwargs is not None:
result = converted_f(*effective_args, **kwargs)
else:
result = converted_f(*effective_args)
return result
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
owner_attr = f
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if logging.has_verbosity(1):
if owner is not None:
composite_desc = '("{}" attr of {})'.format(owner_attr, owner)
else:
composite_desc = ''
logging.log(1,
'Converted call: %s %s\n args: %s\n kwargs: %s\n',
f, composite_desc, args, kwargs)
if inspect_utils.isbuiltin(f):
if kwargs:
return py_builtins.overload_of(f)(*args, **kwargs)
else:
return py_builtins.overload_of(f)(*args)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or
_is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'):
logging.log(2, 'Permanently whitelisted: %s: lru_cache', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if any(f in m.__dict__.values() for m in (collections, pdb, copy, inspect)):
logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
if kwargs is not None:
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self,) + args
else:
effective_args = args
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
effective_args = args
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
effective_args = (f,) + args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
if (not tf_inspect.isclass(target_entity) and
not hasattr(target_entity, '__code__')):
logging.log(
2, 'Permanently whitelisted: %s: native binding', target_entity)
return _call_unconverted(f, args, kwargs)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
arg_values=None,
arg_types=None,
experimental_optional_features=options.optional_features)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
if kwargs is not None:
callargs = tf_inspect.getcallargs(
converted_f, *effective_args, **kwargs)
else:
callargs = tf_inspect.getcallargs(converted_f, *effective_args)
formatted_callargs = '\n'.join(
' {}: {}'.format(k, v) for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
# TODO(mdan): Reduce this list.
except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
ValueError, IOError) as e:
logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True)
if is_autograph_strict_conversion_mode():
raise
logging.warn(
'Entity %s could not be transformed and will be executed as-is.'
' Some features (e.g. tensor-dependent conditionals and loops) may not'
' work as expected.'
' Error details can be found in the logs when running with the env'
' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the'
' AutoGraph team. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
if kwargs is not None:
result = converted_f(*effective_args, **kwargs)
else:
result = converted_f(*effective_args)
return result
def converted_call(f, options, args, kwargs, caller_fn_scope=None):
"""Compiles a function call inline.
For internal use only.
Args:
f: The function to convert.
options: converter.ConversionOptions
args: Tuple, the original positional arguments of f
kwargs: Dict, the original keyword arguments of f
caller_fn_scope: Optional[function_wrappers.FunctionScope], the function
scope of the converted function in which this call was originally made.
Returns:
Any, the result of executing a possibly-converted `f` with the given
arguments.
"""
logging.log(1, 'Converted call: %s\n args: %s\n kwargs: %s\n', f,
args, kwargs)
if conversion.check_cached_unconverted(f, options):
return _call_unconverted(f, args, kwargs, options, False)
if inspect_utils.isbuiltin(f):
if f is eval:
return py_builtins.eval_in_original_context(
f, args, caller_fn_scope)
if f is super:
return py_builtins.super_in_original_context(
f, args, caller_fn_scope)
if kwargs:
return py_builtins.overload_of(f)(*args, **kwargs)
else:
return py_builtins.overload_of(f)(*args)
# TODO(mdan): Clean up the naming inconsistency.
if hasattr(f, 'autograph_info__') or hasattr(f, '__ag_compiled'):
logging.log(2, 'Permanently whitelisted: %s: already converted', f)
return _call_unconverted(f, args, kwargs, options)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper')
or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'{} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will run as-is.'
' You may still apply AutoGraph before the wrapt decorator.'.
format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs, options)
if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'):
logging.log(2, 'Permanently whitelisted: %s: lru_cache', f)
return _call_unconverted(f, args, kwargs, options)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs, options)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, inspect, re)):
logging.log(2, 'Permanently whitelisted: %s: part of builtin module',
f)
return _call_unconverted(f, args, kwargs, options)
# Custom ops and kernels are also permanently whitelisted.
# See tensorflow.framework.load_library.
if (hasattr(f, '__module__')
and hasattr(f.__module__, '_IS_TENSORFLOW_PLUGIN')):
logging.log(2, 'Permanently whitelisted: %s: TensorFlow plugin', f)
return _call_unconverted(f, args, kwargs, options)
if not options.user_requested and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs, options)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs, options)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
# TODO(b/120224672): This unwrapping should be done before the checks above.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
if kwargs is not None:
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self, ) + args
else:
effective_args = args
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
effective_args = (f, ) + args
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
effective_args = args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
if not tf_inspect.isclass(target_entity):
if not hasattr(target_entity, '__code__'):
logging.log(2, 'Permanently whitelisted: %s: native binding',
target_entity)
return _call_unconverted(f, args, kwargs, options)
elif (hasattr(target_entity.__code__, 'co_filename')
and target_entity.__code__.co_filename == '<string>'):
# TODO(mdan): __globals__['txt'] might work in Py3.
logging.log(
2, 'Permanently whitelisted: %s: dynamic code (exec?)',
target_entity)
return _call_unconverted(f, args, kwargs, options)
program_ctx = converter.ProgramContext(
options=options,
autograph_module=tf_inspect.getmodule(converted_call))
converted_f = conversion.convert(target_entity, program_ctx)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
if six.PY3:
logging.log(2, 'KW defaults of %s : %s', converted_f,
converted_f.__kwdefaults__)
if kwargs is not None:
callargs = tf_inspect.getcallargs(converted_f, *effective_args,
**kwargs)
else:
callargs = tf_inspect.getcallargs(converted_f, *effective_args)
formatted_callargs = '\n'.join(' {}: {}'.format(k, v)
for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f,
formatted_callargs)
except Exception as e: # pylint:disable=broad-except
logging.log(1,
'Error transforming entity %s',
target_entity,
exc_info=True)
if is_autograph_strict_conversion_mode():
raise
if _errors_are_normally_possible(target_entity, e):
logging.warn(
'AutoGraph could not transform %s and will run it as-is.\n'
'Cause: %s', target_entity, e)
else:
logging.warn(
'AutoGraph could not transform %s and will run it as-is.\n'
'Please report this to the TensorFlow team. When filing the bug, set'
' the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and'
' attach the full output.\n'
'Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs, options)
with StackTraceMapper(converted_f), tf_stack.CurrentModuleFilter():
try:
if kwargs is not None:
result = converted_f(*effective_args, **kwargs)
else:
result = converted_f(*effective_args)
except Exception as e:
_attach_metadata(e, converted_f, True)
raise
return result
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if options.verbose >= converter.Verbosity.VERBOSE:
logging.info('Converted call: {}; owner: {}'.format(f, owner))
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
# Args typically include `self`, as required by the conversion process.
# When conversion is skipped, `self` is not necessary, because the
# original bound method is being executed. This code removes it.
if tf_inspect.ismethod(f) and args:
f_class = inspect_utils.getmethodclass(f)
if args[0] is f_class:
args = args[1:]
return f(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
# Unwrap functools.partial objects
# TODO(allenl, mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner,) + args
else:
# When the owner is not specified, use the result of
# inspect_utils.getmethodclass.
# TODO(b/119246461): Make sure an owner is always specified.
if not args or args[0] is not f_class:
effective_args = (f_class,) + args
else:
effective_args = (f_class,) + args[1:]
partial_types = (f_class,)
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators,
optional_features=options.optional_features)
result = converted_f(*effective_args, **kwargs)
# The converted function's closure is simply inserted into the function's
# module __dict__. Since modules are permanently cached, that results in
# leaking the entire closure.
# Normally, it's not safe to delete the module because that may release said
# closure as well. However, in the case of converted_call we are certain the
# function will not be executed again, so the closure should no longer be
# needed so long as the function doesn't return any executable code.
# TODO(mdan): Attach the closure properly, using cells.
if all(map(_is_not_callable, nest.flatten(result))):
del sys.modules[converted_f.__module__]
return result
def converted_call(f, owner, options, args, kwargs):
"""Compiles a function call inline. For internal use only."""
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
owner_attr = f
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
if logging.has_verbosity(1):
if owner is not None:
composite_desc = '("{}" attr of {})'.format(owner_attr, owner)
else:
composite_desc = ''
logging.log(1, 'Converted call: %s %s\n args: %s\n kwargs: %s\n',
f, composite_desc, args, kwargs)
if inspect_utils.isbuiltin(f):
if kwargs:
return py_builtins.overload_of(f)(*args, **kwargs)
else:
return py_builtins.overload_of(f)(*args)
# TODO(mdan): Clean up the naming inconsistency.
if hasattr(f, 'autograph_info__') or hasattr(f, '__ag_compiled'):
logging.log(2, 'Permanently whitelisted: %s: already converted', f)
return _call_unconverted(f, args, kwargs)
# TODO(b/122265385): Remove this bypass.
if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper')
or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')):
logging.warn(
'Entity {} appears to be decorated by wrapt, which is not yet supported'
' by AutoGraph. The function will be called without transformation.'
' You may however apply AutoGraph before the decorator.'.format(f))
logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
return _call_unconverted(f, args, kwargs)
if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'):
logging.log(2, 'Permanently whitelisted: %s: lru_cache', f)
return _call_unconverted(f, args, kwargs)
# Constructors are permanently whitelisted.
# TODO(mdan): Toggle as experimental feature instead.
# TODO(b/124016764): Remove this limitation.
if tf_inspect.isclass(f):
logging.log(2, 'Permanently whitelisted: %s: constructor', f)
return _call_unconverted(f, args, kwargs)
# Other built-in modules are permanently whitelisted.
# TODO(mdan): Figure out how to do this consistently for all stdlib modules.
if any(f in m.__dict__.values()
for m in (collections, pdb, copy, inspect, re)):
logging.log(2, 'Permanently whitelisted: %s: part of builtin module',
f)
return _call_unconverted(f, args, kwargs)
# Custom ops and kernels are also permanently whitelisted.
# See tensorflow.framework.load_library.
if (hasattr(f, '__module__')
and hasattr(f.__module__, '_IS_TENSORFLOW_PLUGIN')):
logging.log(2, 'Permanently whitelisted: %s: TensorFlow plugin', f)
return _call_unconverted(f, args, kwargs)
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return _call_unconverted(f, args, kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return _call_unconverted(f, args, kwargs)
# TODO(mdan): Move this entire block inside to_graph.
try: # Begin of transformation error guards
# Unwrap functools.partial objects
# TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
while isinstance(f, functools.partial):
args = f.args + args
new_kwargs = {}
if f.keywords is not None:
new_kwargs.update(f.keywords)
if kwargs is not None:
new_kwargs.update(kwargs)
kwargs = new_kwargs
f = f.func
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
f_self = inspect_utils.getmethodself(f)
# TODO(b/119246461): This may be more elegantly handled using __get__?
if f_self is not None:
effective_args = (f_self, ) + args
else:
effective_args = args
elif tf_inspect.isclass(f):
# Constructors
# Note: Until we support class constructurs, and enable whole-class
# conversion with an experimental flag, this branch is dead code.
# TODO(mdan): Consider removing unless there is a compelling use case.
target_entity = f
effective_args = args
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
effective_args = (f, ) + args
else:
target_entity = f
raise NotImplementedError('unknown callable type "%s"' % type(f))
if not tf_inspect.isclass(target_entity):
if not hasattr(target_entity, '__code__'):
logging.log(2, 'Permanently whitelisted: %s: native binding',
target_entity)
return _call_unconverted(f, args, kwargs)
elif (hasattr(target_entity.__code__, 'co_filename')
and target_entity.__code__.co_filename == '<string>'):
# TODO(mdan): __globals__['txt'] might work in Py3.
logging.log(
2, 'Permanently whitelisted: %s: dynamic code (exec?)',
target_entity)
return _call_unconverted(f, args, kwargs)
converted_f = to_graph(
target_entity,
recursive=options.recursive,
experimental_optional_features=options.optional_features)
if logging.has_verbosity(2):
logging.log(2, 'Defaults of %s : %s', converted_f,
converted_f.__defaults__)
if six.PY3:
logging.log(2, 'KW defaults of %s : %s', converted_f,
converted_f.__kwdefaults__)
if kwargs is not None:
callargs = tf_inspect.getcallargs(converted_f, *effective_args,
**kwargs)
else:
callargs = tf_inspect.getcallargs(converted_f, *effective_args)
formatted_callargs = '\n'.join(' {}: {}'.format(k, v)
for k, v in callargs.items())
logging.log(2, 'Calling %s with\n%s\n', converted_f,
formatted_callargs)
except Exception as e: # pylint:disable=broad-except
logging.log(1,
'Error transforming entity %s',
target_entity,
exc_info=True)
if is_autograph_strict_conversion_mode():
raise
logging.warn(
'Entity %s could not be transformed and will be executed as-is.'
' Please report this to the AutoGraph team. When filing the bug, set'
' the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and'
' attach the full output. Cause: %s', target_entity, e)
return _call_unconverted(f, args, kwargs)
with StackTraceMapper(converted_f), tf_stack.CurrentModuleFilter():
try:
if kwargs is not None:
result = converted_f(*effective_args, **kwargs)
else:
result = converted_f(*effective_args)
except Exception as e:
_attach_metadata(e, converted_f, True)
raise
return result
def converted_call(f, recursive, verbose, force_conversion, arg_types, *args,
**kwargs):
"""Compiles a function call inline. For internal use only."""
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not force_conversion and conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
unknown_arg_value = object() # Sentinel for arguments of unknown value
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
effective_args = args
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
partial_types = (f_class,)
else:
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f,) + args
partial_types = (f.__class__,)
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
for name, arg in arg_values.items():
if arg is unknown_arg_value:
continue
arg_class = arg.__class__
# If arg_value_hints specifies any name, use that instead.
if name not in arg_types:
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__,)
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'],)
converted_f = to_graph(
target_entity,
recursive=recursive,
verbose=verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types)
return converted_f(*effective_args, **kwargs)
def converted_call(f, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
unknown_arg_value = object() # Sentinel for arguments of unknown value
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
effective_args = args
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
partial_types = (f_class, )
else:
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
for name, arg in arg_values.items():
if arg is unknown_arg_value:
continue
arg_class = arg.__class__
# If arg_value_hints specifies any name, use that instead.
if name not in options.arg_types:
options.arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
converted_f = to_graph(target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=options.arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators)
return converted_f(*effective_args, **kwargs)
def converted_call(f, owner, options, *args, **kwargs):
"""Compiles a function call inline. For internal use only."""
if options.verbose:
logging.info('Converted call: {}; owner: {}'.format(f, owner))
if owner is not None:
if not isinstance(f, str):
raise ValueError(
'When owner is specified, the function name must be specified as'
' a string: {}'.format(f))
# Special case when the owner is a 'super' object. In that case lookups of
# dynamic attributes won't work. See
# inspect_utils.SuperWrapperForDynamicAttrs.
if isinstance(owner, super):
owner = inspect_utils.SuperWrapperForDynamicAttrs(owner)
f = getattr(owner, f)
# TODO(mdan): This needs cleanup.
# In particular, we may want to avoid renaming functions altogether.
if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
return f(*args, **kwargs)
if inspect_utils.isbuiltin(f):
return py_builtins.overload_of(f)(*args, **kwargs)
# internal_convert_user_code is for example turned off when issuing a dynamic
# call conversion from generated code while in nonrecursive mode. In that
# case we evidently don't want to recurse, but we still have to convert
# things like builtins.
if not options.internal_convert_user_code:
return f(*args, **kwargs)
if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
# Regular functions
target_entity = f
arg_map_target = f
f_class = inspect_utils.getmethodclass(f)
if f_class is not None:
# If this is a method call, it may or may not include self.
#
# Example when self is included:
# converted_call(to_graph(foo.bar), foo)
#
# Example when self is not included:
# super(...).foo(args)
#
if owner is not None and (not args or args[0] is not owner):
effective_args = (owner, ) + args
else:
effective_args = args
partial_types = (f_class, )
else:
effective_args = args
partial_types = ()
elif tf_inspect.isclass(f):
# Constructors
target_entity = f
arg_map_target = f.__init__
effective_args = args
partial_types = ()
elif hasattr(f, '__call__') and hasattr(f, '__class__'):
# Callable objects
target_entity = f.__call__
arg_map_target = f.__call__
effective_args = (f, ) + args
partial_types = (f.__class__, )
else:
NotImplementedError('unknown callable type "%s"' % type(f))
arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
arg_types = {}
for name, arg in arg_values.items():
arg_class = arg.__class__
arg_types[name] = (arg_class.__name__, arg_class)
# When called from within a decorator, this is the only indication that
# the function is a method - it appears that the decorator is applied
# before the method is bound.
if not partial_types:
if 'self' in arg_values:
if tf_inspect.isclass(arg_values['self'].__class__):
partial_types = (arg_values['self'].__class__, )
elif 'cls' in arg_values:
if tf_inspect.isclass(arg_values['cls']):
partial_types = (arg_values['cls'], )
converted_f = to_graph(target_entity,
recursive=options.recursive,
verbose=options.verbose,
arg_values=arg_values,
arg_types=arg_types,
partial_types=partial_types,
strip_decorators=options.strip_decorators,
optional_features=options.optional_features)
result = converted_f(*effective_args, **kwargs)
# When converting a function, we write a tmp file and import it as a module.
# This leaks the module's closure. Once we've executed the converted_f module
# and there is no more code left to be executed, we can clean up the module.
# TODO(mdan): Look into workarounds that don't suffer from refcount leaks.
# Possibly attach the closure as a regular closure cell, instead of relying on
# module globals.
# If there are callables in the result, they will fail to find their closure
# when called, so only delete module if all returned types are not callable.
flat_results = nest.flatten(result)
if all(map(_is_not_callable, flat_results)):
del sys.modules[converted_f.__module__]
return result
