def _rename_compilable_function(self, node): assert anno.hasanno(node.func, 'live_val') assert anno.hasanno(node.func, 'fqn') target_entity = anno.getanno(node.func, 'live_val') target_fqn = anno.getanno(node.func, 'fqn') if not self._should_compile(node, target_fqn): return node if anno.hasanno(node, 'is_constructor'): new_name = self.context.namer.compiled_class_name( target_fqn, live_entity=target_entity) do_rename = True else: owner_type = self._determine_function_owner(target_entity) new_name, do_rename = self.context.namer.compiled_function_name( target_fqn, live_entity=target_entity, owner_type=owner_type) if do_rename: if target_entity is not None: if tf_inspect.ismethod(target_entity): # The renaming process will transform it into a regular function. # TODO(mdan): Is this complete? How does it work with nested members? node.args = [node.func.value] + node.args node.func = gast.Name(new_name, gast.Load(), None) return node
def fn_args(fn): """Get argument names for function-like object. Args: fn: Function, or function-like object (e.g., result of `functools.partial`). Returns: `tuple` of string argument names. Raises: ValueError: if partial function has positionally bound arguments """ _, fn = tf_decorator.unwrap(fn) # Handle callables. if hasattr(fn, '__call__') and tf_inspect.ismethod(fn.__call__): return tuple(tf_inspect.getargspec(fn.__call__).args) # Handle functools.partial and similar objects. if hasattr(fn, 'func') and hasattr(fn, 'keywords') and hasattr(fn, 'args'): # Handle nested partial. original_args = fn_args(fn.func) if not original_args: return tuple() return tuple([ arg for arg in original_args[len(fn.args):] if arg not in set((fn.keywords or {}).keys()) ]) # Handle function. return tuple(tf_inspect.getargspec(fn).args)
def convert_inline(f, *args, **kwargs): """Shorthand to convert and call a function. For example, the following two statements are equivalent: @convert() def foo(): ... foo(bar) def foo(): ... convert_inline(foo, bar) Args: f: Function to convert. Only this call will be converted. *args: Passed through to f. **kwargs: Passed through to f, with the following exceptions: * arg_value_hints: A dict mapping parameter names to objects that can hint at the type of those parameters. Returns: The result of the converted f applied to args and kwargs. """ if 'arg_value_hints' in kwargs: arg_value_hints = kwargs['arg_value_hints'] del kwargs['arg_value_hints'] else: arg_value_hints = None if tf_inspect.ismethod(f): # When converting methods, the result is still an unbound function. args = (f.__self__,) + args return convert(arg_value_hints)(f)(*args, **kwargs)
def _is_known_loaded_type(f, module_name, entity_name): """Tests whether the function or method is an instance of a known type.""" if (module_name not in sys.modules or not hasattr(sys.modules[module_name], entity_name)): return False type_entity = getattr(sys.modules[module_name], entity_name) if isinstance(f, type_entity): # The method if of this type. Example: # # o = ClassType() # function(o.method)() return True if tf_inspect.ismethod(f): f = six.get_unbound_function(f) # The the unbound method if of this type. Example: # # class ClassType: # @function # def method(self): # ... # o = ClassType() # o.method() if isinstance(f, type_entity): return True return False
def _rename_compilable_function(self, node): assert anno.hasanno(node.func, 'live_val') assert anno.hasanno(node.func, 'fqn') target_entity = anno.getanno(node.func, 'live_val') target_fqn = anno.getanno(node.func, 'fqn') if not self._should_compile(node, target_fqn): return node if anno.hasanno(node, 'is_constructor'): new_name = self.ctx.namer.compiled_class_name( target_fqn, live_entity=target_entity) do_rename = True else: if anno.hasanno(node.func, 'parent_type'): owner_type = anno.getanno(node.func, 'parent_type') else: # Fallback - not reliable. owner_type = inspect_utils.getmethodclass(target_entity) new_name, do_rename = self.ctx.namer.compiled_function_name( target_fqn, live_entity=target_entity, owner_type=owner_type) if do_rename: if target_entity is not None: if tf_inspect.ismethod(target_entity): # The renaming process will transform it into a regular function. # TODO(mdan): Is this complete? How does it work with nested members? node.args = [node.func.value] + node.args node.func = templates.replace('func_name', func_name=new_name)[0] return node
def __call__(self, func): # Various sanity checks on the callable func. if not callable(func): raise ValueError("func %s must be callable" % func) # Func should not use kwargs and defaults. argspec = tf_inspect.getargspec(func) if argspec.keywords or argspec.defaults: raise ValueError("Functions with argument defaults or keyword " "arguments are not supported.") # Computes how many arguments 'func' has. min_args = len(argspec.args) max_args = min_args if argspec.varargs: max_args = 1000000 argnames = argspec.args if tf_inspect.ismethod(func): # 1st argument is the "class" type. min_args -= 1 argnames = argnames[1:] if self._input_types: # If Defun is given a list of types for the inputs, the number # of input types should be compatible with 'func'. num = len(self._input_types) if num < min_args or num > max_args: raise ValueError( "The function has fewer arguments than the number of specified " "input types.") return _DefinedFunction( func, argnames, self._input_types, self._func_name, self._grad_func, self._python_grad_func, out_names=self._out_names, **self._extra_kwargs) # 'func' expects no arguments and input types is an empty list. if min_args == 0 and max_args == 0: return _DefinedFunction( func, [], [], self._func_name, self._grad_func, self._python_grad_func, out_names=self._out_names, **self._extra_kwargs) # Input types are unknown. It's an overloaded function and hence # its definition needs to be deferred until it's called. return _OverloadedFunction( func, argnames, self._func_name, self._grad_func, self._python_grad_func, out_names=self._out_names, **self._extra_kwargs)
def class_to_graph(c, conversion_map): """Specialization of `entity_to_graph` for classes.""" converted_members = {} method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m) members = tf_inspect.getmembers(c, predicate=method_filter) if not members: raise ValueError('Cannot convert %s: it has no member methods.' % c) class_namespace = None for _, m in members: node, _ = function_to_graph( m, conversion_map=conversion_map, arg_values={}, arg_types={'self': (c.__name__, c)}, owner_type=c) # TODO(mdan): Do not assume all members have the same view of globals. if class_namespace is None: class_namespace = inspect_utils.getnamespace(m) converted_members[m] = node namer = conversion_map.new_namer(class_namespace) class_name = namer.compiled_class_name(c.__name__, c) node = gast.ClassDef( class_name, bases=[], keywords=[], body=list(converted_members.values()), decorator_list=[]) return node, class_name
def entity_to_graph(o, conversion_map, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. conversion_map: A ConversionMap object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, conversion_map) elif tf_inspect.isfunction(o): node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) conversion_map.add_to_cache(o, node) if conversion_map.recursive: while True: candidate = None for obj in conversion_map.name_map.keys(): if obj not in conversion_map.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr(candidate, 'im_class') not in conversion_map.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, conversion_map, {}, {}) return node, name, ns
def _verify_metric_fn_args(metric_fn): args = set(estimator_util.fn_args(metric_fn)) if tf_inspect.ismethod(metric_fn): if 'self' in args: args.remove('self') invalid_args = list(args - _VALID_METRIC_FN_ARGS) if invalid_args: raise ValueError('metric_fn (%s) has following not expected args: %s' % (metric_fn, invalid_args))
def _get_func_name(func): _, func = tf_decorator.unwrap(func) if callable(func): if tf_inspect.isfunction(func): return func.__name__ elif tf_inspect.ismethod(func): return "%s.%s" % (func.__self__.__name__, func.__name__) else: # Probably a class instance with __call__ return type(func) else: raise ValueError("Argument must be callable")
def get_func_code(func): """Returns func_code of passed callable.""" _, func = tf_decorator.unwrap(func) if callable(func): if tf_inspect.isfunction(func) or tf_inspect.ismethod(func): return six.get_function_code(func) elif hasattr(func, '__call__'): return six.get_function_code(func.__call__) else: raise ValueError('Unhandled callable, type=%s' % type(func)) else: raise ValueError('Argument must be callable')
def getfutureimports(entity): """Detects what future imports are necessary to safely execute entity source. Args: entity: Any object Returns: A tuple of future strings """ if not (tf_inspect.isfunction(entity) or tf_inspect.ismethod(entity)): return tuple() return tuple(sorted(name for name, value in entity.__globals__.items() if getattr(value, '__module__', None) == '__future__'))
def get_func_name(func): """Returns name of passed callable.""" _, func = tf_decorator.unwrap(func) if callable(func): if tf_inspect.isfunction(func): return func.__name__ elif tf_inspect.ismethod(func): return '%s.%s' % (six.get_method_self(func).__class__.__name__, six.get_method_function(func).__name__) else: # Probably a class instance with __call__ return str(type(func)) else: raise ValueError('Argument must be callable')
def convert_entity_to_ast(o, program_ctx): """Compile a Python entity into equivalent TensorFlow. Args: o: A Python entity. program_ctx: A ProgramContext object. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ logging.log(1, 'Converting %s', o) if tf_inspect.isclass(o): nodes, name, entity_info = convert_class_to_ast(o, program_ctx) elif tf_inspect.isfunction(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) elif tf_inspect.ismethod(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) # TODO(mdan,yashkatariya): Remove when object conversion is implemented. elif hasattr(o, '__class__'): raise NotImplementedError( 'Object conversion is not yet supported. If you are ' 'trying to convert code that uses an existing object, ' 'try including the creation of that object in the ' 'conversion. For example, instead of converting the method ' 'of a class, try converting the entire class instead. ' 'See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/' 'python/autograph/README.md#using-the-functional-api ' 'for more information.') else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) if logging.has_verbosity(2): logging.log(2, 'Compiled output of %s:\n\n%s\n', o, compiler.ast_to_source(nodes)) if logging.has_verbosity(4): for n in nodes: logging.log(4, 'Compiled AST of %s:\n\n%s\n\n', o, pretty_printer.fmt(n, color=False)) return nodes, name, entity_info
def _instantiate(entity, converted_entity_info, free_nonglobal_var_names): """Creates a converted instance and binds it to match original entity.""" factory = converted_entity_info.get_factory() # `factory` is currently bound to the empty module it was loaded from. # It must instead be bound to the globals and closure from the original # entity. if tf_inspect.isfunction(entity) or tf_inspect.ismethod(entity): entity_globals = entity.__globals__ entity_closure = entity.__closure__ or () elif hasattr(entity, '__module__'): entity_globals = sys.modules[entity.__module__].__dict__ entity_closure = () assert len(entity_closure) == len(free_nonglobal_var_names) # Fit the original entity's cells to match the order of factory's cells. original_names_and_cells = dict(zip(free_nonglobal_var_names, entity_closure)) new_factory_cells = tuple( original_names_and_cells[name] for name in factory.__code__.co_freevars) bound_factory = types.FunctionType( code=factory.__code__, globals=entity_globals, name=factory.__name__, argdefs=(), closure=new_factory_cells) # Two other free vars: the internal "ag__" module and the source # map. These are wired via the parameters of the factory. converted_entity = bound_factory( # pylint:disable=not-callable ag_internal, converted_entity_info.source_map, converted_entity_info.get_module()) if tf_inspect.isfunction(entity) or tf_inspect.ismethod(entity): # Attach the default argument to the converted function. converted_entity.__defaults__ = entity.__defaults__ return converted_entity
def object_to_graph(o, conversion_map, value_hints): """Compile a Python object into equivalent TensorFlow. The function will also recursively compile all the objects that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python object. conversion_map: A ConversionMap object. value_hints: A dict containing value hints for symbols like function parameters. Returns: A tuple (ast, new_name): * ast: An AST representing an object with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new object can be found. Raises: ValueError: if the object is not supported. """ if value_hints is None: value_hints = {} if tf_inspect.isclass(o): node, new_name = class_to_graph(o, conversion_map, value_hints) elif tf_inspect.isfunction(o): node, new_name = function_to_graph(o, conversion_map, value_hints) elif tf_inspect.ismethod(o): node, new_name = function_to_graph(o, conversion_map, value_hints) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) conversion_map.add_to_cache(o, node) if conversion_map.recursive: for obj in conversion_map.name_map.keys(): if obj not in conversion_map.dependency_cache: if (hasattr(obj, 'im_class') and getattr(obj, 'im_class') not in conversion_map.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue object_to_graph(obj, conversion_map, None) return node, new_name
def get_func_code(func): """Returns func_code of passed callable, or None if not available.""" _, func = tf_decorator.unwrap(func) if callable(func): if tf_inspect.isfunction(func) or tf_inspect.ismethod(func): return six.get_function_code(func) # Since the object is not a function or method, but is a callable, we will # try to access the __call__method as a function. This works with callable # classes but fails with functool.partial objects despite their __call__ # attribute. try: return six.get_function_code(func.__call__) except AttributeError: return None else: raise ValueError('Argument must be callable')
def convert_entity_to_ast(o, program_ctx): """Compile a Python entity into equivalent TensorFlow. Args: o: A Python entity. program_ctx: A ProgramContext object. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ logging.log(1, 'Converting %s', o) if tf_inspect.isclass(o): nodes, name, entity_info = convert_class_to_ast(o, program_ctx) elif tf_inspect.isfunction(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) elif tf_inspect.ismethod(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) elif hasattr(o, '__class__'): # Note: this should only be raised when attempting to convert the object # directly. converted_call should still support it. raise NotImplementedError( 'cannot convert entity "{}": object conversion is not yet' ' supported.'.format(o)) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) if logging.has_verbosity(2): logging.log(2, 'Compiled output of %s:\n\n%s\n', o, compiler.ast_to_source(nodes)) if logging.has_verbosity(4): for n in nodes: logging.log(4, 'Compiled AST of %s:\n\n%s\n\n', o, pretty_printer.fmt(n, color=False)) return nodes, name, entity_info
def _get_raw_docstring(py_object): """Get the docs for a given python object. Args: py_object: A python object to retrieve the docs for (class, function/method, or module). Returns: The docstring, or the empty string if no docstring was found. """ # For object instances, tf_inspect.getdoc does give us the docstring of their # type, which is not what we want. Only return the docstring if it is useful. if (tf_inspect.isclass(py_object) or tf_inspect.ismethod(py_object) or tf_inspect.isfunction(py_object) or tf_inspect.ismodule(py_object) or isinstance(py_object, property)): return tf_inspect.getdoc(py_object) or '' else: return ''
def convert(entity, program_ctx): """Converts an entity into an equivalent entity.""" if tf_inspect.isfunction(entity) or tf_inspect.ismethod(entity): free_nonglobal_var_names = entity.__code__.co_freevars else: free_nonglobal_var_names = () for i, name in enumerate(free_nonglobal_var_names): if (name == 'ag__' and entity.__closure__[i].cell_contents is not ag_internal): raise ValueError('entity {} uses the reserved symbol "{}"'.format( entity, name)) # TODO(mdan): In extreme cases, other ag__ symbols may also be clobbered. converted_entity_info = _convert_with_cache( entity, program_ctx, free_nonglobal_var_names) return _instantiate(entity, converted_entity_info, free_nonglobal_var_names)
def _verify_model_fn_args(model_fn, params): """Verifies model fn arguments.""" args = set(util.fn_args(model_fn)) if 'features' not in args: raise ValueError('model_fn (%s) must include features argument.' % model_fn) if params is not None and 'params' not in args: raise ValueError('model_fn (%s) does not include params argument, ' 'but params (%s) is passed to Estimator.' % (model_fn, params)) if params is None and 'params' in args: logging.warning('Estimator\'s model_fn (%s) includes params ' 'argument, but params are not passed to Estimator.', model_fn) if tf_inspect.ismethod(model_fn): if 'self' in args: args.remove('self') non_valid_args = list(args - _VALID_MODEL_FN_ARGS) if non_valid_args: raise ValueError('model_fn (%s) has following not expected args: %s' % (model_fn, non_valid_args))
def converted_call(f, recursive, verbose, arg_types, *args, **kwargs): """Compiles a function call inline.""" # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) unknown_arg_value = object() # Sentinel for arguments of unknown value if inspect_utils.isbuiltin(f): return builtins.dynamic_builtin(f, *args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f effective_args = args f_class = inspect_utils.getmethodclass(f) if f_class is not None: partial_types = (f_class,) else: partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f,) + args partial_types = (f.__class__,) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) for name, arg in arg_values.items(): if arg is unknown_arg_value: continue arg_class = arg.__class__ # If arg_value_hints specifies any name, use that instead. if name not in arg_types: arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__,) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'],) converted_f = to_graph( target_entity, recursive=recursive, verbose=verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types) return converted_f(*effective_args, **kwargs)
def entity_to_graph(o, program_ctx, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. program_ctx: A ProgramContext object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if program_ctx.options.verbose == converter.Verbosity.VERBOSE: logging.info('Converting {}'.format(o)) if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, program_ctx) elif tf_inspect.isfunction(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) # TODO(mdan,yashkatariya): Remove when object conversion is implemented. elif hasattr(o, '__class__'): raise NotImplementedError( 'Object conversion is not yet supported. If you are ' 'trying to convert code that uses an existing object, ' 'try including the creation of that object in the ' 'conversion. For example, instead of converting the method ' 'of a class, try converting the entire class instead. ' 'See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/' 'contrib/autograph/README.md#using-the-functional-api ' 'for more information.') else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) # TODO(mdan): This is temporary. it should be created using a converter. # TODO(mdan): The attribute should be added with a helper, not directly. # The helper can ensure there are no collisions. template = ''' entity.autograph_info__ = {} ''' node.extend(templates.replace(template, entity=name)) program_ctx.add_to_cache(o, node) if program_ctx.options.verbose == converter.Verbosity.VERBOSE: logging.info('Compiled output of {}:\n\n{}\n'.format( o, compiler.ast_to_source(node))) if program_ctx.options.recursive: while True: candidate = None for obj in program_ctx.name_map.keys(): if obj not in program_ctx.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr(candidate, 'im_class') not in program_ctx.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, program_ctx, {}, {}) return node, name, ns
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" logging.log(1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f, owner, args, kwargs) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) # TODO(b/122265385): Remove this bypass. if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f)) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return _call_unconverted(f, args, kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return _call_unconverted(f, args, kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. # Note: TF linter disallows importing inspect. if any(f in m.__dict__.values() for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return _call_unconverted(f, args, kwargs) if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return _call_unconverted(f, args, kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return _call_unconverted(f, args, kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner,) + args else: # When the owner is not specified, use the result of # inspect_utils.getmethodclass. # TODO(b/119246461): Make sure an owner is always specified. if not args or args[0] is not f_self: effective_args = (f_self,) + args else: effective_args = (f_self,) + args[1:] partial_types = (f_self,) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f,) + args partial_types = (f.__class__,) else: raise NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__,) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'],) logging.log(3, 'Partial types in conversion of %s: %s', target_entity, partial_types) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=arg_values, arg_types=arg_types, experimental_optional_features=options.optional_features, experimental_strip_decorators=options.strip_decorators, experimental_verbose=options.verbose, experimental_partial_types=partial_types) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) formatted_callargs = '\n'.join( ' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) # TODO(mdan): Reduce this list. except (errors.AutoGraphError, AssertionError, AttributeError, IndexError, KeyError, NameError, NotImplementedError, SyntaxError, TypeError, ValueError, IOError) as e: logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) logging.warn( 'Entity %s could not be transformed and will be staged without change.' ' Error details can be found in the logs when running with the env' ' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the' ' AutoGraph team. Cause: %s', target_entity, e) return _call_unconverted(f, args, kwargs) result = converted_f(*effective_args, **kwargs) # The converted function's closure is simply inserted into the function's # module __dict__. Since modules are permanently cached, that results in # leaking the entire closure. # Normally, it's not safe to delete the module because that may release said # closure as well. However, in the case of converted_call we are certain the # function will not be executed again, so the closure should no longer be # needed so long as the function doesn't return any executable code. # TODO(mdan): Attach the closure properly, using cells. if all(map(_is_not_callable, nest.flatten(result))): del sys.modules[converted_f.__module__] return result
def docs_for_object(full_name, py_object, parser_config): """Return a PageInfo object describing a given object from the TF API. This function uses _parse_md_docstring to parse the docs pertaining to `object`. This function resolves '@{symbol}' references in the docstrings into links to the appropriate location. It also adds a list of alternative names for the symbol automatically. It assumes that the docs for each object live in a file given by `documentation_path`, and that relative links to files within the documentation are resolvable. Args: full_name: The fully qualified name of the symbol to be documented. py_object: The Python object to be documented. Its documentation is sourced from `py_object`'s docstring. parser_config: A ParserConfig object. Returns: Either a `_FunctionPageInfo`, `_ClassPageInfo`, or a `_ModulePageInfo` depending on the type of the python object being documented. Raises: RuntimeError: If an object is encountered for which we don't know how to make docs. """ # Which other aliases exist for the object referenced by full_name? master_name = parser_config.reference_resolver.py_master_name(full_name) duplicate_names = parser_config.duplicates.get(master_name, [full_name]) # TODO(wicke): Once other pieces are ready, enable this also for partials. if (tf_inspect.ismethod(py_object) or tf_inspect.isfunction(py_object) or # Some methods in classes from extensions come in as routines. tf_inspect.isroutine(py_object)): page_info = _FunctionPageInfo(master_name) page_info.set_signature(py_object, parser_config.reverse_index) elif tf_inspect.isclass(py_object): page_info = _ClassPageInfo(master_name) page_info.collect_docs_for_class(py_object, parser_config) elif tf_inspect.ismodule(py_object): page_info = _ModulePageInfo(master_name) page_info.collect_docs_for_module(parser_config) else: raise RuntimeError('Cannot make docs for object %s: %r' % (full_name, py_object)) relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(full_name)) or '.') page_info.set_doc( _parse_md_docstring(py_object, relative_path, parser_config.reference_resolver)) page_info.set_aliases(duplicate_names) page_info.set_guides( _get_guides_markdown(duplicate_names, parser_config.guide_index, relative_path)) page_info.set_defined_in(_get_defined_in(py_object, parser_config)) return page_info
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) owner_attr = f # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if logging.has_verbosity(1): if owner is not None: composite_desc = '("{}" attr of {})'.format(owner_attr, owner) else: composite_desc = '' logging.log(1, 'Converted call: %s %s\n args: %s\n kwargs: %s\n', f, composite_desc, args, kwargs) if inspect_utils.isbuiltin(f): if kwargs: return py_builtins.overload_of(f)(*args, **kwargs) else: return py_builtins.overload_of(f)(*args) # TODO(b/122265385): Remove this bypass. if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f)) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return _call_unconverted(f, args, kwargs) if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'): logging.log(2, 'Permanently whitelisted: %s: lru_cache', f) return _call_unconverted(f, args, kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return _call_unconverted(f, args, kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. if any(f in m.__dict__.values() for m in (collections, pdb, copy, inspect)): logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return _call_unconverted(f, args, kwargs) if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return _call_unconverted(f, args, kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return _call_unconverted(f, args, kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) if kwargs is not None: new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: effective_args = (f_self,) + args else: effective_args = args elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f effective_args = args elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ effective_args = (f,) + args else: target_entity = f raise NotImplementedError('unknown callable type "%s"' % type(f)) if (not tf_inspect.isclass(target_entity) and not hasattr(target_entity, '__code__')): logging.log( 2, 'Permanently whitelisted: %s: native binding', target_entity) return _call_unconverted(f, args, kwargs) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=None, arg_types=None, experimental_optional_features=options.optional_features) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) if kwargs is not None: callargs = tf_inspect.getcallargs( converted_f, *effective_args, **kwargs) else: callargs = tf_inspect.getcallargs(converted_f, *effective_args) formatted_callargs = '\n'.join( ' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) # TODO(mdan): Reduce this list. except (errors.AutoGraphError, AssertionError, AttributeError, IndexError, KeyError, NameError, NotImplementedError, SyntaxError, TypeError, ValueError, IOError) as e: logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) if is_autograph_strict_conversion_mode(): raise logging.warn( 'Entity %s could not be transformed and will be executed as-is.' ' Some features (e.g. tensor-dependent conditionals and loops) may not' ' work as expected.' ' Error details can be found in the logs when running with the env' ' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the' ' AutoGraph team. Cause: %s', target_entity, e) return _call_unconverted(f, args, kwargs) if kwargs is not None: result = converted_f(*effective_args, **kwargs) else: result = converted_f(*effective_args) return result
def converted_call(f, owner, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) unknown_arg_value = object() # Sentinel for arguments of unknown value if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_class = inspect_utils.getmethodclass(f) if f_class is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner,) + args else: effective_args = args partial_types = (f_class,) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f,) + args partial_types = (f.__class__,) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): if arg is unknown_arg_value: continue arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__,) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'],) converted_f = to_graph( target_entity, recursive=options.recursive, verbose=options.verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types, strip_decorators=options.strip_decorators) return converted_f(*effective_args, **kwargs)
def convert_class_to_ast(c, program_ctx): """Specialization of `convert_entity_to_ast` for classes.""" # TODO(mdan): Revisit this altogether. Not sure we still need it. converted_members = {} method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m ) members = tf_inspect.getmembers(c, predicate=method_filter) if not members: raise ValueError('cannot convert %s: no member methods' % c) # TODO(mdan): Don't clobber namespaces for each method in one class namespace. # The assumption that one namespace suffices for all methods only holds if # all methods were defined in the same module. # If, instead, functions are imported from multiple modules and then spliced # into the class, then each function has its own globals and __future__ # imports that need to stay separate. # For example, C's methods could both have `global x` statements referring to # mod1.x and mod2.x, but using one namespace for C would cause a conflict. # from mod1 import f1 # from mod2 import f2 # class C(object): # method1 = f1 # method2 = f2 class_namespace = {} future_features = None for _, m in members: # Only convert the members that are directly defined by the class. if inspect_utils.getdefiningclass(m, c) is not c: continue (node, ), _, entity_info = convert_func_to_ast(m, program_ctx=program_ctx, do_rename=False) class_namespace.update(entity_info.namespace) converted_members[m] = node # TODO(mdan): Similarly check the globals. if future_features is None: future_features = entity_info.future_features elif frozenset(future_features) ^ frozenset( entity_info.future_features): # Note: we can support this case if ever needed. raise ValueError( 'cannot convert {}: if has methods built with mismatched future' ' features: {} and {}'.format(c, future_features, entity_info.future_features)) namer = naming.Namer(class_namespace) class_name = namer.class_name(c.__name__) # Process any base classes: if the superclass if of a whitelisted type, an # absolute import line is generated. output_nodes = [] renames = {} base_names = [] for base in c.__bases__: if isinstance(object, base): base_names.append('object') continue if is_whitelisted_for_graph(base): alias = namer.new_symbol(base.__name__, ()) output_nodes.append( gast.ImportFrom( module=base.__module__, names=[gast.alias(name=base.__name__, asname=alias)], level=0)) else: raise NotImplementedError( 'Conversion of classes that do not directly extend classes from' ' whitelisted modules is temporarily suspended. If this breaks' ' existing code please notify the AutoGraph team immediately.') base_names.append(alias) renames[qual_names.QN(base.__name__)] = qual_names.QN(alias) # Generate the definition of the converted class. bases = [gast.Name(n, gast.Load(), None) for n in base_names] class_def = gast.ClassDef(class_name, bases=bases, keywords=[], body=list(converted_members.values()), decorator_list=[]) # Make a final pass to replace references to the class or its base classes. # Most commonly, this occurs when making super().__init__() calls. # TODO(mdan): Making direct references to superclass' superclass will fail. class_def = qual_names.resolve(class_def) renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name) class_def = ast_util.rename_symbols(class_def, renames) output_nodes.append(class_def) # TODO(mdan): Find a way better than forging this object. entity_info = transformer.EntityInfo(source_code=None, source_file=None, future_features=future_features, namespace=class_namespace) return output_nodes, class_name, entity_info
def converted_call(f, owner, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" if options.verbose: logging.info('Converted call: {}; owner: {}'.format(f, owner)) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) unknown_arg_value = object() # Sentinel for arguments of unknown value if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_class = inspect_utils.getmethodclass(f) if f_class is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner, ) + args else: effective_args = args partial_types = (f_class, ) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f, ) + args partial_types = (f.__class__, ) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): if arg is unknown_arg_value: continue arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__, ) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'], ) converted_f = to_graph(target_entity, recursive=options.recursive, verbose=options.verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types, strip_decorators=options.strip_decorators, optional_features=options.optional_features) return converted_f(*effective_args, **kwargs)
def converted_call(f, owner, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" if options.verbose: logging.info('Converted call: {}; owner: {}'.format(f, owner)) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return f(*args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_class = inspect_utils.getmethodclass(f) if f_class is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner, ) + args else: effective_args = args partial_types = (f_class, ) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f, ) + args partial_types = (f.__class__, ) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__, ) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'], ) converted_f = to_graph(target_entity, recursive=options.recursive, verbose=options.verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types, strip_decorators=options.strip_decorators, optional_features=options.optional_features) result = converted_f(*effective_args, **kwargs) # When converting a function, we write a tmp file and import it as a module. # This leaks the module's closure. Once we've executed the converted_f module # and there is no more code left to be executed, we can clean up the module. # TODO(mdan): Look into workarounds that don't suffer from refcount leaks. # Possibly attach the closure as a regular closure cell, instead of relying on # module globals. # If there are callables in the result, they will fail to find their closure # when called, so only delete module if all returned types are not callable. flat_results = nest.flatten(result) if all(map(_is_not_callable, flat_results)): del sys.modules[converted_f.__module__] return result
def __call__(self, func): # Various sanity checks on the callable func. if not callable(func): raise ValueError("function %s must be callable" % func) # Func should not use kwargs and defaults. argspec = tf_inspect.getargspec(func) if argspec.keywords or argspec.defaults: raise ValueError( "function with argument defaults or keywords arguments are not" " supported. {} has defaults {} and keywords {}.".format( func, argspec.defaults, argspec.keywords)) # Computes how many arguments 'func' has. min_args = len(argspec.args) max_args = min_args if argspec.varargs: max_args = 1000000 argnames = argspec.args if tf_inspect.ismethod(func): # 1st argument is the "class" type. min_args -= 1 argnames = argnames[1:] if self._input_types: # If Defun is given a list of types for the inputs, the number # of input types should be compatible with 'func'. num = len(self._input_types) if num < min_args or num > max_args: raise ValueError( "The function has fewer arguments than the number of specified " "input types.") return _DefinedFunction( func, argnames, self._input_types, self._func_name, self._grad_func, self._python_grad_func, out_names=self._out_names, **self._extra_kwargs) # 'func' expects no arguments and input types is an empty list. if min_args == 0 and max_args == 0: return _DefinedFunction( func, [], [], self._func_name, self._grad_func, self._python_grad_func, out_names=self._out_names, **self._extra_kwargs) # Input types are unknown. It's an overloaded function and hence # its definition needs to be deferred until it's called. return _OverloadedFunction( func, argnames, self._func_name, self._grad_func, self._python_grad_func, out_names=self._out_names, **self._extra_kwargs)
def class_to_graph(c, conversion_map): """Specialization of `entity_to_graph` for classes.""" converted_members = {} method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m ) members = tf_inspect.getmembers(c, predicate=method_filter) if not members: raise ValueError('Cannot convert %s: it has no member methods.' % c) class_namespace = {} for _, m in members: # Only convert the members that are directly defined by the class. if inspect_utils.getdefiningclass(m, c) is not c: continue node, _, namespace = function_to_graph( m, conversion_map=conversion_map, arg_values={}, arg_types={'self': (c.__name__, c)}, owner_type=c) if class_namespace is None: class_namespace = namespace else: class_namespace.update(namespace) converted_members[m] = node namer = conversion_map.new_namer(class_namespace) class_name = namer.compiled_class_name(c.__name__, c) # TODO (mdan): This needs to be explained more thoroughly. id:673 # https://github.com/imdone/tensorflow/issues/674 # Process any base classes: if the sueprclass if of a whitelisted type, an # absolute import line is generated. Otherwise, it is marked for conversion # (as a side effect of the call to namer.compiled_class_name() followed by # conversion_map.update_name_map(namer)). output_nodes = [] renames = {} bases = [] for base in c.__bases__: if isinstance(object, base): bases.append('object') continue if is_whitelisted_for_graph(base): alias = namer.new_symbol(base.__name__, ()) output_nodes.append( gast.ImportFrom( module=base.__module__, names=[gast.alias(name=base.__name__, asname=alias)], level=0)) else: # This will trigger a conversion into a class with this name. alias = namer.compiled_class_name(base.__name__, base) bases.append(alias) renames[qual_names.QN(base.__name__)] = qual_names.QN(alias) conversion_map.update_name_map(namer) # Generate the definition of the converted class. output_nodes.append( gast.ClassDef(class_name, bases=bases, keywords=[], body=list(converted_members.values()), decorator_list=[])) node = gast.Module(output_nodes) # Make a final pass to replace references to the class or its base classes. # Most commonly, this occurs when making super().__init__() calls. # TODO (mdan): Making direct references to superclass' superclass will fail. id:521 # https://github.com/imdone/tensorflow/issues/522 node = qual_names.resolve(node) renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name) node = ast_util.rename_symbols(node, renames) return node, class_name, class_namespace
def entity_to_graph(o, conversion_map, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. conversion_map: A ConversionMap object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, conversion_map) elif tf_inspect.isfunction(o): # TODO (mdan): This is not a reliable mechanism. id:619 # https://github.com/imdone/tensorflow/issues/620 # The most reliable way is to check the source code, the AST will contain # a Lambda node instead of a FunctionDef if o.__name__ == '<lambda>': raise NotImplementedError( 'lambda functions are not yet supported; declare the function' ' using def instead: %s' % o) else: node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) conversion_map.add_to_cache(o, node) if conversion_map.recursive: while True: candidate = None for obj in conversion_map.name_map.keys(): if obj not in conversion_map.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr(candidate, 'im_class') not in conversion_map.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, conversion_map, {}, {}) return node, name, ns
def testIsMethod(self): self.assertTrue(tf_inspect.ismethod(TestDecoratedClass().two)) self.assertFalse(tf_inspect.ismethod(test_decorated_function))
def converted_call(f, recursive, verbose, arg_types, *args, **kwargs): """Compiles a function call inline.""" # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) unknown_arg_value = object() # Sentinel for arguments of unknown value if tf_inspect.isbuiltin(f): return builtins.dynamic_builtin(f, *args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f effective_args = args f_class = inspect_utils.getmethodclass(f) if f_class is not None: partial_types = (f_class, ) else: partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = (unknown_arg_value, ) + args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f, ) + args partial_types = (f.__class__, ) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) for name, arg in arg_values.items(): if arg is unknown_arg_value: continue arg_class = arg.__class__ # If arg_value_hints specifies any name, use that instead. if name not in arg_types: arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__, ) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'], ) converted_f = to_graph(target_entity, recursive=recursive, verbose=verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types) return converted_f(*effective_args, **kwargs)
def is_whitelisted_for_graph(o, check_call_override=True): """Checks whether an entity is whitelisted for use in graph mode. Examples of whitelisted entities include all members of the tensorflow package. Args: o: A Python entity. check_call_override: Reserved for internal use. When set to `False`, it disables the rule according to which classes are whitelisted if their __call__ method is whitelisted. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) if hasattr(m, '__name__'): # Builtins typically have unnamed modules. for prefix, in config.DEFAULT_UNCOMPILED_MODULES: if m.__name__.startswith(prefix + '.') or m.__name__ == prefix: logging.log(2, 'Whitelisted: %s: name starts with "%s"', o, prefix) return True if hasattr(o, 'autograph_info__') or hasattr(o, '__ag_compiled'): logging.log(2, 'Whitelisted: %s: already converted', o) return True if tf_inspect.isgeneratorfunction(o): logging.warn( 'Entity {} appears to be a generator function. It will not be converted' ' by AutoGraph.'.format(o), 1) logging.log(2, 'Whitelisted: %s: generator functions are not converted', o) return True if check_call_override and hasattr(o, '__call__'): # Callable objects: whitelisted if their __call__ method is. # The type check avoids infinite recursion around the __call__ method # of function objects. if (type(o) != type(o.__call__)) and is_whitelisted_for_graph( o.__call__): # pylint: disable=unidiomatic-typecheck logging.log(2, 'Whitelisted: %s: object __call__ whitelisted', o) return True owner_class = None if tf_inspect.ismethod(o): # Methods of whitelisted classes are also whitelisted, even if they are # bound via user subclasses. # # For example, suppose `tf.Foo` has a method called `bar`, and `baz` is # defined as below. `tf.Foo` is whitelisted. Then `baz.bar` is also # whitelisted. # # class Custom(tf.Foo): # pass # # baz = Custom() # # For the example above, if `Custom` did overload `bar`, then it would no # longer be whitelisted. owner_class = inspect_utils.getmethodclass(o) if owner_class is not None: if issubclass(owner_class, unittest.TestCase): logging.log(2, 'Whitelisted: %s: method of TestCase subclass', o) return True owner_class = inspect_utils.getdefiningclass(o, owner_class) is_call_override = (o.__name__ == '__call__') if is_whitelisted_for_graph( owner_class, check_call_override=not is_call_override): logging.log(2, 'Whitelisted: %s: owner is whitelisted %s', o, owner_class) return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if tf_inspect.isclass(o) and len(o.__bases__) > 1: logging.warn( 'Entity {} looks like a namedtuple subclass. Its constructor will' ' not be converted by AutoGraph, but if it has any custom methods,' ' those will be.'.format(o), 1) logging.log(2, 'Whitelisted: %s: named tuple', o) return True logging.log(2, 'Not whitelisted: %s: default rule', o) return False
def _is_callable_object(obj): return hasattr(obj, '__call__') and tf_inspect.ismethod(obj.__call__)
def converted_call(f, owner, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" if options.verbose >= converter.Verbosity.VERBOSE: logging.info('Converted call: {}; owner: {}'.format(f, owner)) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): # Args typically include `self`, as required by the conversion process. # When conversion is skipped, `self` is not necessary, because the # original bound method is being executed. This code removes it. if tf_inspect.ismethod(f) and args: f_class = inspect_utils.getmethodclass(f) if args[0] is f_class: args = args[1:] return f(*args, **kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return f(*args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_class = inspect_utils.getmethodclass(f) # TODO(mdan): This may be more elegantly handled using __get__? if f_class is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner, ) + args else: # Always override the self arg, because it might be different from # what the method was bound to - see inspect_utils.getmethodclass. assert args, 'Bound function call without self argument?' effective_args = (f_class, ) + args[1:] partial_types = (f_class, ) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f, ) + args partial_types = (f.__class__, ) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__, ) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'], ) converted_f = to_graph(target_entity, recursive=options.recursive, verbose=options.verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types, strip_decorators=options.strip_decorators, optional_features=options.optional_features) result = converted_f(*effective_args, **kwargs) # The converted function's closure is simply inserted into the function's # module __dict__. Since modules are permanently cached, that results in # leaking the entire closure. # Normally, it's not safe to delete the module because that may release said # closure as well. However, in the case of converted_call we are certain the # function will not be executed again, so the closure should no longer be # needed so long as the function doesn't return any executable code. # TODO(mdan): Attach the closure properly, using cells. if all(map(_is_not_callable, nest.flatten(result))): del sys.modules[converted_f.__module__] return result
def _is_bounded_method(fn): _, fn = tf_decorator.unwrap(fn) return tf_inspect.ismethod(fn) and (fn.__self__ is not None)
def entity_to_graph(o, program_ctx, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. program_ctx: A ProgramContext object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, program_ctx) elif tf_inspect.isfunction(o): # TODO(mdan): This is not a reliable mechanism. # The most reliable way is to check the source code, the AST will contain # a Lambda node instead of a FunctionDef if o.__name__ == '<lambda>': raise NotImplementedError( 'lambda functions are not yet supported; declare the function' ' using def instead: %s' % o) else: node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) # TODO(mdan): This is temporary. it should be created using a converter. # TODO(mdan): The attribute should be added with a helper, not directly. # The helper can ensure there are no collisions. template = ''' entity.autograph_info__ = {} ''' node.extend(templates.replace(template, entity=name)) program_ctx.add_to_cache(o, node) if program_ctx.recursive: while True: candidate = None for obj in program_ctx.name_map.keys(): if obj not in program_ctx.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr(candidate, 'im_class') not in program_ctx.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, program_ctx, {}, {}) return node, name, ns
def is_allowlisted(o, check_call_override=True, allow_namedtuple_subclass=False): """Checks whether an entity is allowed for use in graph mode. Examples of allowed entities include all members of the tensorflow package. Args: o: A Python entity. check_call_override: Reserved for internal use. When set to `False`, it disables the rule according to which classes are allowed if their __call__ method is allowed. allow_namedtuple_subclass: Reserved for internal use. When `True`, namedtuple subclasses are not allowed. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) # Examples of callables that lack a __module__ property include builtins. if hasattr(m, '__name__'): for rule in config.CONVERSION_RULES: action = rule.get_action(m) if action == config.Action.CONVERT: logging.log(2, 'Not allowed: %s: %s', o, rule) return False elif action == config.Action.DO_NOT_CONVERT: logging.log(2, 'Allowlisted: %s: %s', o, rule) return True # The check for __code__ below is because isgeneratorfunction crashes # without one. if hasattr(o, '__code__') and tf_inspect.isgeneratorfunction(o): logging.log(2, 'Allowlisted: %s: generator functions are not converted', o) return True if (check_call_override and not tf_inspect.isclass(o) and hasattr(o, '__call__')): # Callable objects: allowed if their __call__ method is. # The type check avoids infinite recursion around the __call__ method # of function objects. if (type(o) != type(o.__call__)) and is_allowlisted(o.__call__): # pylint: disable=unidiomatic-typecheck logging.log(2, 'Allowlisted: %s: object __call__ allowed', o) return True owner_class = None if tf_inspect.ismethod(o): # Methods of allowed classes are also allowed, even if they are # bound via user subclasses. # # For example, suppose `tf.Foo` has a method called `bar`, and `baz` is # defined as below. `tf.Foo` is allowed. Then `baz.bar` is also # allowed. # # class Custom(tf.Foo): # pass # # baz = Custom() # # For the example above, if `Custom` did overload `bar`, then it would no # longer be allowed. owner_class = inspect_utils.getmethodclass(o) if owner_class is function.TfMethodTarget: owner_class = o.__self__.target_class if owner_class is not None: if issubclass(owner_class, unittest.TestCase): logging.log(2, 'Allowlisted: %s: method of TestCase subclass', o) return True owner_class = inspect_utils.getdefiningclass(o, owner_class) if is_allowlisted(owner_class, check_call_override=False, allow_namedtuple_subclass=True): logging.log(2, 'Allowlisted: %s: owner is allowed %s', o, owner_class) return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if allow_namedtuple_subclass: if not any( inspect_utils.isnamedtuple(base) for base in o.__bases__): logging.log(2, 'Allowlisted: %s: named tuple', o) return True else: logging.log(2, 'Allowlisted: %s: named tuple or subclass', o) return True logging.log(2, 'Not allowed: %s: default rule', o) return False
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" logging.log(1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f, owner, args, kwargs) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) # TODO(b/122265385): Remove this bypass. if ('wrapt' in sys.modules and hasattr(sys.modules['wrapt'], 'FunctionWrapper') and isinstance(f, sys.modules['wrapt'].FunctionWrapper)): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f), 1) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return f(*args, **kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return f(*args, **kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. # Note: TF linter disallows importing inspect. if any(f in m.__dict__.values() for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return f(*args, **kwargs) # TODO(mdan): This needs cleanup. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): # TODO(mdan): This may be inconsistent in certain situations. # If the function had already been annotated with @tf.function, it # may be bound to the incorrect object. It's unclear if those situations # are possible, but if they happen, we need to check if f is bound # to a shim like WeakrefSelf and unpack it. # Args typically include `self`, as required by the conversion process. # When conversion is skipped, `self` is not necessary, because the # original bound method is being executed. This code removes it. if tf_inspect.ismethod(f) and args: f_self = inspect_utils.getmethodself(f) if args[0] is f_self: args = args[1:] return f(*args, **kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return f(*args, **kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner,) + args else: # When the owner is not specified, use the result of # inspect_utils.getmethodclass. # TODO(b/119246461): Make sure an owner is always specified. if not args or args[0] is not f_self: effective_args = (f_self,) + args else: effective_args = (f_self,) + args[1:] partial_types = (f_self,) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f,) + args partial_types = (f.__class__,) else: raise NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__,) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'],) logging.log(3, 'Partial types in conversion of %s: %s', target_entity, partial_types) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=arg_values, arg_types=arg_types, experimental_optional_features=options.optional_features, experimental_strip_decorators=options.strip_decorators, experimental_verbose=options.verbose, experimental_partial_types=partial_types) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) formatted_callargs = '\n'.join( ' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) # TODO(mdan): Reduce this list. except (errors.AutoGraphError, AssertionError, AttributeError, IndexError, KeyError, NameError, NotImplementedError, SyntaxError, TypeError, ValueError, IOError) as e: logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) logging.warn( 'Entity %s could not be transformed and will be staged without change.' ' Error details can be found in the logs when running with the env' ' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the' ' AutoGraph team. Cause: %s', target_entity, e) return f(*args, **kwargs) result = converted_f(*effective_args, **kwargs) # The converted function's closure is simply inserted into the function's # module __dict__. Since modules are permanently cached, that results in # leaking the entire closure. # Normally, it's not safe to delete the module because that may release said # closure as well. However, in the case of converted_call we are certain the # function will not be executed again, so the closure should no longer be # needed so long as the function doesn't return any executable code. # TODO(mdan): Attach the closure properly, using cells. if all(map(_is_not_callable, nest.flatten(result))): del sys.modules[converted_f.__module__] return result
def entity_to_graph(o, program_ctx, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. program_ctx: A ProgramContext object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, program_ctx) elif tf_inspect.isfunction(o): # TODO(mdan): This is not a reliable mechanism. # The most reliable way is to check the source code, the AST will contain # a Lambda node instead of a FunctionDef if o.__name__ == '<lambda>': raise NotImplementedError( 'lambda functions are not yet supported; declare the function' ' using def instead: %s' % o) else: node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) # TODO(mdan,yashkatariya): Remove when object conversion is implemented. elif hasattr(o, '__class__'): raise NotImplementedError( 'Object conversion is not yet supported. If you are ' 'trying to convert code that uses an existing object, ' 'try including the creation of that object in the ' 'conversion. For example, instead of converting the method ' 'of a class, try converting the entire class instead. ' 'See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/' 'contrib/autograph/README.md#using-the-functional-api ' 'for more information.') else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) # TODO(mdan): This is temporary. it should be created using a converter. # TODO(mdan): The attribute should be added with a helper, not directly. # The helper can ensure there are no collisions. template = ''' entity.autograph_info__ = {} ''' node.extend(templates.replace(template, entity=name)) program_ctx.add_to_cache(o, node) if program_ctx.recursive: while True: candidate = None for obj in program_ctx.name_map.keys(): if obj not in program_ctx.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr( candidate, 'im_class') not in program_ctx.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, program_ctx, {}, {}) return node, name, ns
def collect_docs_for_class(self, py_class, parser_config): """Collect information necessary specifically for a class's doc page. Mainly, this is details about information about the class's members. Args: py_class: the class object being documented parser_config: An instance of ParserConfig. """ doc_path = documentation_path(self.full_name) relative_path = os.path.relpath(path='.', start=os.path.dirname(doc_path) or '.') for short_name in parser_config.tree[self.full_name]: # Remove builtin members that we never want to document. if short_name in [ '__class__', '__base__', '__weakref__', '__doc__', '__module__', '__dict__', '__abstractmethods__', '__slots__', '__getnewargs__' ]: continue child_name = '.'.join([self.full_name, short_name]) child = parser_config.py_name_to_object(child_name) # Don't document anything that is defined in object or by protobuf. defining_class = _get_defining_class(py_class, short_name) if (defining_class is object or defining_class is type or defining_class is tuple or defining_class is BaseException or defining_class is Exception or # The following condition excludes most protobuf-defined symbols. defining_class and defining_class.__name__ in ['CMessage', 'Message', 'MessageMeta']): continue # TODO(markdaoust): Add a note in child docs showing the defining class. child_doc = _parse_md_docstring(child, relative_path, parser_config.reference_resolver) if isinstance(child, property): self._add_property(short_name, child_name, child, child_doc) elif tf_inspect.isclass(child): if defining_class is None: continue url = parser_config.reference_resolver.reference_to_url( child_name, relative_path) self._add_class(short_name, child_name, child, child_doc, url) elif (tf_inspect.ismethod(child) or tf_inspect.isfunction(child) or tf_inspect.isroutine(child)): if defining_class is None: continue # Omit methods defined by namedtuple. original_method = defining_class.__dict__[short_name] if (hasattr(original_method, '__module__') and (original_method.__module__ or '').startswith('namedtuple')): continue # Some methods are often overridden without documentation. Because it's # obvious what they do, don't include them in the docs if there's no # docstring. if not child_doc.brief.strip() and short_name in [ '__str__', '__repr__', '__hash__', '__del__', '__copy__' ]: print('Skipping %s, defined in %s, no docstring.' % (child_name, defining_class)) continue try: child_signature = _generate_signature( child, parser_config.reverse_index) except TypeError: # If this is a (dynamically created) slot wrapper, tf_inspect will # raise typeerror when trying to get to the code. Ignore such # functions. continue self._add_method(short_name, child_name, child, child_doc, child_signature) else: # Exclude members defined by protobuf that are useless if issubclass(py_class, ProtoMessage): if (short_name.endswith('_FIELD_NUMBER') or short_name in ['__slots__', 'DESCRIPTOR']): continue # TODO(wicke): We may want to also remember the object itself. self._add_other_member(short_name, child_name, child, child_doc)
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" logging.log( 1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f, owner, args, kwargs) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) if _is_known_loaded_type(f, 'weakref', 'ref'): logging.log(2, 'Permanently whitelisted: %s: weakref', f) return _call_unconverted(f, args, kwargs) # TODO(b/122265385): Remove this bypass. if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f)) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return _call_unconverted(f, args, kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return _call_unconverted(f, args, kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. # Note: TF linter disallows importing inspect. if any(f in m.__dict__.values() for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return _call_unconverted(f, args, kwargs) if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return _call_unconverted(f, args, kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return _call_unconverted(f, args, kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: effective_args = (f_self, ) + args else: effective_args = args elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f effective_args = args elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ effective_args = (f, ) + args else: target_entity = f raise NotImplementedError('unknown callable type "%s"' % type(f)) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=None, arg_types=None, experimental_optional_features=options.optional_features) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) formatted_callargs = '\n'.join(' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) # TODO(mdan): Reduce this list. except (errors.AutoGraphError, AssertionError, AttributeError, IndexError, KeyError, NameError, NotImplementedError, SyntaxError, TypeError, ValueError, IOError) as e: logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) if is_autograph_strict_conversion_mode(): raise logging.warn( 'Entity %s could not be transformed and will be executed as-is.' ' Some features (e.g. tensor-dependent conditionals and loops) may not' ' work as expected.' ' Error details can be found in the logs when running with the env' ' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the' ' AutoGraph team. Cause: %s', target_entity, e) return _call_unconverted(f, args, kwargs) result = converted_f(*effective_args, **kwargs) return result
def is_whitelisted_for_graph(o): """Check whether an entity is whitelisted for use in graph mode. Examples of whitelisted entities include all members of the tensorflow package. Args: o: A Python entity. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) if not hasattr(m, '__name__'): # Note: typically it's builtins that fall in this category. Builtins will # be handled by specific code that follows this screening layer. logging.log(2, '%s is NOT whitelisted: unknown module name', o) return False for prefix, in config.DEFAULT_UNCOMPILED_MODULES: if m.__name__.startswith(prefix): logging.log(2, '%s is whitelisted: name starts with "%s"', o, prefix) return True if hasattr(o, 'autograph_info__') or hasattr(o, '__ag_compiled'): logging.log(2, '%s is whitelisted: already converted', o) return True if (not inspect_utils.isweakrefself(o) and not tf_inspect.isclass(o) and hasattr(o, '__call__') and hasattr(o, '__class__')): # Callable objects: whitelisted if their __call__ method is. call_whitelisted = is_whitelisted_for_graph(o.__call__) if call_whitelisted: logging.log(2, '%s is whitelisted: object __call__ whitelisted', o) return call_whitelisted if tf_inspect.ismethod(o): # Methods of whitelisted classes are also whitelisted, even if they are # bound via user subclasses. # # For example, suppose `tf.Foo` has a method called `bar`, and `baz` is # defined as below. `tf.Foo` is whitelisted. Then `baz.bar` is also # whitelisted. # # class Custom(tf.Foo): # pass # # baz = Custom() # # For the example above, if `Custom` did overload `bar`, then it would no # longer be whitelisted. owner_class = inspect_utils.getmethodclass(o) if owner_class is not None: owner_class = inspect_utils.getdefiningclass(o, owner_class) if is_whitelisted_for_graph(owner_class): logging.log(2, '%s is whitelisted: owner is whitelisted %s', o, owner_class) return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if tf_inspect.isclass(o) and len(o.__bases__) > 1: logging.warn_first_n( 'Entity {} looks like a namedtuple subclass. If it has any custom' ' methods, they will not be converted by AutoGraph.'.format(o), 1) logging.log(2, '%s is whitelisted: named tuple', o) return True logging.log(2, '%s is NOT whitelisted', o) return False
def converted_call(f, owner, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" logging.log( 1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f, owner, args, kwargs) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. if f in collections.__dict__.values(): return f(*args, **kwargs) # TODO(mdan): This needs cleanup. if (not options.force_conversion and conversion.is_whitelisted_for_graph(f)): # TODO(mdan): This may be inconsistent in certain situations. # If the function had already been annotated with @tf.function, it # may be bound to the incorrect object. It's unclear if those situations # are possible, but if they happen, we need to check if f is bound # to a shim like WeakrefSelf and unpack it. # Args typically include `self`, as required by the conversion process. # When conversion is skipped, `self` is not necessary, because the # original bound method is being executed. This code removes it. if tf_inspect.ismethod(f) and args: f_self = inspect_utils.getmethodself(f) if args[0] is f_self: args = args[1:] return f(*args, **kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return f(*args, **kwargs) # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_self = inspect_utils.getmethodself(f) if owner is not None: partial_types = (type(owner), ) elif f_self is not None: partial_types = (type(f_self), ) else: partial_types = () # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner, ) + args else: # When the owner is not specified, use the result of # inspect_utils.getmethodclass. # TODO(b/119246461): Make sure an owner is always specified. if not args or args[0] is not f_self: effective_args = (f_self, ) + args else: effective_args = (f_self, ) + args[1:] else: effective_args = args elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f, ) + args partial_types = (f.__class__, ) else: raise NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__, ) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'], ) logging.log(3, 'Partial types in conversion of %s: %s', target_entity, partial_types) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=arg_values, arg_types=arg_types, experimental_optional_features=options.optional_features, experimental_strip_decorators=options.strip_decorators, experimental_verbose=options.verbose, experimental_partial_types=partial_types) if logging.has_verbosity(2): callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) formatted_callargs = '\n'.join(' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) result = converted_f(*effective_args, **kwargs) # The converted function's closure is simply inserted into the function's # module __dict__. Since modules are permanently cached, that results in # leaking the entire closure. # Normally, it's not safe to delete the module because that may release said # closure as well. However, in the case of converted_call we are certain the # function will not be executed again, so the closure should no longer be # needed so long as the function doesn't return any executable code. # TODO(mdan): Attach the closure properly, using cells. if all(map(_is_not_callable, nest.flatten(result))): del sys.modules[converted_f.__module__] return result
def class_to_graph(c, program_ctx): """Specialization of `entity_to_graph` for classes.""" converted_members = {} method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m) members = tf_inspect.getmembers(c, predicate=method_filter) if not members: raise ValueError('Cannot convert %s: it has no member methods.' % c) class_namespace = {} for _, m in members: # Only convert the members that are directly defined by the class. if inspect_utils.getdefiningclass(m, c) is not c: continue node, _, namespace = function_to_graph( m, program_ctx=program_ctx, arg_values={}, arg_types={'self': (c.__name__, c)}, owner_type=c) if class_namespace is None: class_namespace = namespace else: class_namespace.update(namespace) converted_members[m] = node[0] namer = program_ctx.new_namer(class_namespace) class_name = namer.compiled_class_name(c.__name__, c) # TODO(mdan): This needs to be explained more thoroughly. # Process any base classes: if the superclass if of a whitelisted type, an # absolute import line is generated. Otherwise, it is marked for conversion # (as a side effect of the call to namer.compiled_class_name() followed by # program_ctx.update_name_map(namer)). output_nodes = [] renames = {} base_names = [] for base in c.__bases__: if isinstance(object, base): base_names.append('object') continue if is_whitelisted_for_graph(base): alias = namer.new_symbol(base.__name__, ()) output_nodes.append( gast.ImportFrom( module=base.__module__, names=[gast.alias(name=base.__name__, asname=alias)], level=0)) else: # This will trigger a conversion into a class with this name. alias = namer.compiled_class_name(base.__name__, base) base_names.append(alias) renames[qual_names.QN(base.__name__)] = qual_names.QN(alias) program_ctx.update_name_map(namer) # Generate the definition of the converted class. bases = [gast.Name(n, gast.Load(), None) for n in base_names] class_def = gast.ClassDef( class_name, bases=bases, keywords=[], body=list(converted_members.values()), decorator_list=[]) # Make a final pass to replace references to the class or its base classes. # Most commonly, this occurs when making super().__init__() calls. # TODO(mdan): Making direct references to superclass' superclass will fail. class_def = qual_names.resolve(class_def) renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name) class_def = ast_util.rename_symbols(class_def, renames) output_nodes.append(class_def) return output_nodes, class_name, class_namespace
def converted_call(f, args, kwargs, caller_fn_scope=None, options=None): """Compiles a function call inline. For internal use only. Note: The argument list is optimized for readability of generated code, which may look like this: ag__.converted_call(f, (arg1, arg2), None, fscope) ag__.converted_call(f, (), dict(arg1=val1, **kwargs), fscope) ag__.converted_call(f, (arg1, arg2) + varargs, dict(**kwargs), lscope) Args: f: The function to convert. args: Tuple, the original positional arguments of f kwargs: Optional[Dict], the original keyword arguments of f caller_fn_scope: Optional[function_wrappers.FunctionScope], the function scope of the converted function in which this call was originally made. options: Optional[converter.ConversionOptions], conversion options. If not specified, the value of caller_fn_scope.callopts is used. Either options or caller_fn_scope must be present. Returns: Any, the result of executing a possibly-converted `f` with the given arguments. """ logging.log(1, 'Converted call: %s\n args: %s\n kwargs: %s\n', f, args, kwargs) if options is None: if caller_fn_scope is None: raise ValueError('either caller_fn_scope or options must have a value') options = caller_fn_scope.callopts if conversion.check_cached_unconverted(f, options): return _call_unconverted(f, args, kwargs, options, False) if inspect_utils.isbuiltin(f): if f is eval: return py_builtins.eval_in_original_context(f, args, caller_fn_scope) if f is super: return py_builtins.super_in_original_context(f, args, caller_fn_scope) if kwargs: return py_builtins.overload_of(f)(*args, **kwargs) else: return py_builtins.overload_of(f)(*args) # TODO(mdan): Clean up the naming inconsistency. if hasattr(f, 'autograph_info__') or hasattr(f, '__ag_compiled'): logging.log(2, 'Permanently whitelisted: %s: already converted', f) return _call_unconverted(f, args, kwargs, options) # TODO(b/122265385): Remove this bypass. if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')): logging.warn( '{} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will run as-is.' ' You may still apply AutoGraph before the wrapt decorator.'.format(f)) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return _call_unconverted(f, args, kwargs, options) if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'): logging.log(2, 'Permanently whitelisted: %s: lru_cache', f) return _call_unconverted(f, args, kwargs, options) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return _call_unconverted(f, args, kwargs, options) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. if any( f in m.__dict__.values() for m in (collections, pdb, copy, inspect, re)): logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return _call_unconverted(f, args, kwargs, options) # Custom ops and kernels are also permanently whitelisted. # See tensorflow.framework.load_library. if (hasattr(f, '__module__') and hasattr(f.__module__, '_IS_TENSORFLOW_PLUGIN')): logging.log(2, 'Permanently whitelisted: %s: TensorFlow plugin', f) return _call_unconverted(f, args, kwargs, options) if not options.user_requested and conversion.is_whitelisted_for_graph(f): return _call_unconverted(f, args, kwargs, options) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return _call_unconverted(f, args, kwargs, options) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. # TODO(b/120224672): This unwrapping should be done before the checks above. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) if kwargs is not None: new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: effective_args = (f_self,) + args else: effective_args = args elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ effective_args = (f,) + args elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f effective_args = args else: target_entity = f raise NotImplementedError('unknown callable type "%s"' % type(f)) if not tf_inspect.isclass(target_entity): if not hasattr(target_entity, '__code__'): logging.log(2, 'Permanently whitelisted: %s: native binding', target_entity) return _call_unconverted(f, args, kwargs, options) elif (hasattr(target_entity.__code__, 'co_filename') and target_entity.__code__.co_filename == '<string>'): # TODO(mdan): __globals__['txt'] might work in Py3. logging.log(2, 'Permanently whitelisted: %s: dynamic code (exec?)', target_entity) return _call_unconverted(f, args, kwargs, options) program_ctx = converter.ProgramContext( options=options, autograph_module=tf_inspect.getmodule(converted_call)) converted_f = conversion.convert(target_entity, program_ctx) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) if six.PY3: logging.log(2, 'KW defaults of %s : %s', converted_f, converted_f.__kwdefaults__) if kwargs is not None: callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) else: callargs = tf_inspect.getcallargs(converted_f, *effective_args) formatted_callargs = '\n'.join( ' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) except Exception as e: # pylint:disable=broad-except logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) if is_autograph_strict_conversion_mode(): raise if _errors_are_normally_possible(target_entity, e): logging.warn( 'AutoGraph could not transform %s and will run it as-is.\n' 'Cause: %s', target_entity, e) else: logging.warn( 'AutoGraph could not transform %s and will run it as-is.\n' 'Please report this to the TensorFlow team. When filing the bug, set' ' the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and' ' attach the full output.\n' 'Cause: %s', target_entity, e) return _call_unconverted(f, args, kwargs, options) with StackTraceMapper(converted_f), tf_stack.CurrentModuleFilter(): try: if kwargs is not None: result = converted_f(*effective_args, **kwargs) else: result = converted_f(*effective_args) except Exception as e: _attach_metadata(e, converted_f, True) raise return result
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) owner_attr = f # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if logging.has_verbosity(1): if owner is not None: composite_desc = '("{}" attr of {})'.format(owner_attr, owner) else: composite_desc = '' logging.log(1, 'Converted call: %s %s\n args: %s\n kwargs: %s\n', f, composite_desc, args, kwargs) if inspect_utils.isbuiltin(f): if kwargs: return py_builtins.overload_of(f)(*args, **kwargs) else: return py_builtins.overload_of(f)(*args) # TODO(mdan): Clean up the naming inconsistency. if hasattr(f, 'autograph_info__') or hasattr(f, '__ag_compiled'): logging.log(2, 'Permanently whitelisted: %s: already converted', f) return _call_unconverted(f, args, kwargs) # TODO(b/122265385): Remove this bypass. if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f)) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return _call_unconverted(f, args, kwargs) if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'): logging.log(2, 'Permanently whitelisted: %s: lru_cache', f) return _call_unconverted(f, args, kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return _call_unconverted(f, args, kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. if any(f in m.__dict__.values() for m in (collections, pdb, copy, inspect, re)): logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return _call_unconverted(f, args, kwargs) # Custom ops and kernels are also permanently whitelisted. # See tensorflow.framework.load_library. if (hasattr(f, '__module__') and hasattr(f.__module__, '_IS_TENSORFLOW_PLUGIN')): logging.log(2, 'Permanently whitelisted: %s: TensorFlow plugin', f) return _call_unconverted(f, args, kwargs) if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return _call_unconverted(f, args, kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return _call_unconverted(f, args, kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) if kwargs is not None: new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: effective_args = (f_self, ) + args else: effective_args = args elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f effective_args = args elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ effective_args = (f, ) + args else: target_entity = f raise NotImplementedError('unknown callable type "%s"' % type(f)) if not tf_inspect.isclass(target_entity): if not hasattr(target_entity, '__code__'): logging.log(2, 'Permanently whitelisted: %s: native binding', target_entity) return _call_unconverted(f, args, kwargs) elif (hasattr(target_entity.__code__, 'co_filename') and target_entity.__code__.co_filename == '<string>'): # TODO(mdan): __globals__['txt'] might work in Py3. logging.log( 2, 'Permanently whitelisted: %s: dynamic code (exec?)', target_entity) return _call_unconverted(f, args, kwargs) converted_f = to_graph( target_entity, recursive=options.recursive, experimental_optional_features=options.optional_features) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) if six.PY3: logging.log(2, 'KW defaults of %s : %s', converted_f, converted_f.__kwdefaults__) if kwargs is not None: callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) else: callargs = tf_inspect.getcallargs(converted_f, *effective_args) formatted_callargs = '\n'.join(' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) except Exception as e: # pylint:disable=broad-except logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) if is_autograph_strict_conversion_mode(): raise logging.warn( 'Entity %s could not be transformed and will be executed as-is.' ' Please report this to the AutoGraph team. When filing the bug, set' ' the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and' ' attach the full output. Cause: %s', target_entity, e) return _call_unconverted(f, args, kwargs) try: with StackTraceMapper(converted_f), tf_stack.CurrentModuleFilter(): if kwargs is not None: result = converted_f(*effective_args, **kwargs) else: result = converted_f(*effective_args) except Exception as e: _attach_metadata(e, converted_f, True) raise return result