def _traverse_internal(root, visit, stack, path): """Internal helper for traverse.""" # Only traverse modules and classes if not tf_inspect.isclass(root) and not tf_inspect.ismodule(root): return try: children = tf_inspect.getmembers(root) # Add labels for duplicate values in Enum. if tf_inspect.isclass(root) and issubclass(root, enum.Enum): for enum_member in root.__members__.items(): if enum_member not in children: children.append(enum_member) children = sorted(children) except ImportError: # On some Python installations, some modules do not support enumerating # members (six in particular), leading to import errors. children = [] new_stack = stack + [root] visit(path, root, children) for name, child in children: # Do not descend into built-in modules if tf_inspect.ismodule( child) and child.__name__ in sys.builtin_module_names: continue # Break cycles if any(child is item for item in new_stack): # `in`, but using `is` continue child_path = path + '.' + name if path else name _traverse_internal(child, visit, new_stack, child_path)
def compiled_function_name(self, original_name, live_object=None, owner_type=None): """See call_trees.FunctionNamer.compiled_function_name.""" if live_object is not None and live_object in self.renamed_calls: return self.renamed_calls[live_object] if not self.recursive: new_name = original_name elif owner_type is None or owner_type in self.partial_types: # Top level functions: rename new_name_root = 'tf__%s' % original_name new_name = new_name_root n = 0 while new_name in self.global_namespace: n += 1 new_name = '%s_%d' % (new_name_root, n) else: if tf_inspect.isclass(owner_type): # Class members: do not rename (the entire class will be renamed) new_name = original_name else: raise NotImplementedError('Member function "%s" of non-class type: %s' % (original_name, owner_type)) if live_object is not None: self.renamed_calls[live_object] = new_name self.generated_names.add(new_name) return new_name
def _process_variable_assignment(self, source, targets): # Special case: constructors. if isinstance(source, gast.Call): func = source.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): anno.setanno(source, 'is_constructor', True) anno.setanno(source, 'type', func_obj) anno.setanno(source, 'type_fqn', anno.getanno(func, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. # Multiple targets mean multiple assignment. for target in targets: # Tuple target means unpacking. if isinstance(target, gast.Tuple): for i, target_item in enumerate(target.elts): # Two cases here: # 1. Static unpacking, e.g. a, b = c, d # 2. Dynamic unpacking, e.g. a, b = c # The former case is optimized away. if isinstance(source, (gast.Tuple, gast.List)): source_item = source.elts[i] else: source_item = gast.Subscript(source, gast.Index(i), ctx=None) self._process_variable_assignment(source_item, (target_item,)) elif isinstance(target, (gast.Name, gast.Attribute)): target_symbol = anno.getanno(target, anno.Basic.QN) self.scope.setval(target_symbol, source) else: raise ValueError( 'assignment target has unknown type: %s' % target_item)
def _process_variable_assignment(self, source, targets): if isinstance(source, gast.Call): func = source.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): anno.setanno(source, 'is_constructor', True) anno.setanno(source, 'type', func_obj) anno.setanno(source, 'type_fqn', anno.getanno(func, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. for t in targets: if isinstance(t, gast.Tuple): for i, e in enumerate(t.elts): self.scope.setval(e.id, gast.Subscript( source, gast.Index(i), ctx=gast.Store())) elif isinstance(t, gast.Name): self.scope.setval(t.id, source) elif isinstance(t, gast.Attribute): if not (isinstance(t.value, gast.Name) and t.value.id == 'self'): raise ValueError( 'Dont know how to handle assignment to attributes of objects' ' other than "self": [%s].%s' % (t.value, t.attr)) else: raise ValueError('Dont know how to handle assignment to %s' % t)
def from_visitor(cls, visitor, doc_index, **kwargs): """A factory function for building a ReferenceResolver from a visitor. Args: visitor: an instance of `DocGeneratorVisitor` doc_index: a dictionary mapping document names to references objects with "title" and "url" fields **kwargs: all remaining args are passed to the constructor Returns: an instance of `ReferenceResolver` () """ is_class = { name: tf_inspect.isclass(visitor.index[name]) for name, obj in visitor.index.items() } is_module = { name: tf_inspect.ismodule(visitor.index[name]) for name, obj in visitor.index.items() } return cls( duplicate_of=visitor.duplicate_of, doc_index=doc_index, is_class=is_class, is_module=is_module, **kwargs)
def __call__(self, parent_name, parent, children): """Visitor interface, see `tensorflow/tools/common:traverse` for details. This method is called for each symbol found in a traversal using `tensorflow/tools/common:traverse`. It should not be called directly in user code. Args: parent_name: The fully qualified name of a symbol found during traversal. parent: The Python object referenced by `parent_name`. children: A list of `(name, py_object)` pairs enumerating, in alphabetical order, the children (as determined by `tf_inspect.getmembers`) of `parent`. `name` is the local name of `py_object` in `parent`. Raises: RuntimeError: If this visitor is called with a `parent` that is not a class or module. """ parent_name = self._add_prefix(parent_name) self._index[parent_name] = parent self._tree[parent_name] = [] if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)): raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name, parent)) for i, (name, child) in enumerate(list(children)): # Don't document __metaclass__ if name in ['__metaclass__']: del children[i] continue full_name = '.'.join([parent_name, name]) if parent_name else name self._index[full_name] = child self._tree[parent_name].append(name)
def is_supported_type_for_deprecation(symbol): # Exclude Exception subclasses since users should be able to # "except" the same type of exception that was "raised" (i.e. we # shouldn't wrap it with deprecation alias). # Also, exclude subclasses of namedtuples for now. return tf_inspect.isfunction(symbol) or ( tf_inspect.isclass(symbol) and not issubclass(symbol, Exception) and tuple not in tf_inspect.getmro(symbol))
def __call__(self, path, parent, children): # The path to the object. lib_path = 'tensorflow.%s' % path if path else 'tensorflow' # A small helper method to construct members(children) protos. def _AddMember(member_name, member_obj, proto): """Add the child object to the object being constructed.""" _, member_obj = tf_decorator.unwrap(member_obj) if member_name == '__init__' or not member_name.startswith('_'): if tf_inspect.isroutine(member_obj): new_method = proto.member_method.add() new_method.name = member_name # If member_obj is a python builtin, there is no way to get its # argspec, because it is implemented on the C side. It also has no # func_code. if getattr(member_obj, 'func_code', None): new_method.argspec = _SanitizedArgSpec(member_obj) else: new_member = proto.member.add() new_member.name = member_name new_member.mtype = str(type(member_obj)) parent_corner_cases = _CORNER_CASES.get(path, {}) if path not in _CORNER_CASES or parent_corner_cases: # Decide if we have a module or a class. if tf_inspect.ismodule(parent): # Create a module object. module_obj = api_objects_pb2.TFAPIModule() for name, child in children: if name in parent_corner_cases: # If we have an empty entry, skip this object. if parent_corner_cases[name]: module_obj.member.add(**(parent_corner_cases[name])) else: _AddMember(name, child, module_obj) # Store the constructed module object. self._protos[lib_path] = api_objects_pb2.TFAPIObject( path=lib_path, tf_module=module_obj) elif tf_inspect.isclass(parent): # Construct a class. class_obj = api_objects_pb2.TFAPIClass() class_obj.is_instance.extend(_SanitizedMRO(parent)) for name, child in children: if name in parent_corner_cases: # If we have an empty entry, skip this object. if parent_corner_cases[name]: module_obj.member.add(**(parent_corner_cases[name])) else: _AddMember(name, child, class_obj) # Store the constructed class object. self._protos[lib_path] = api_objects_pb2.TFAPIObject( path=lib_path, tf_class=class_obj) else: logging.error('Illegal call to ApiProtoDump::_py_obj_to_proto.' 'Object is neither a module nor a class: %s', path)
def entity_to_graph(o, conversion_map, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. conversion_map: A ConversionMap object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, conversion_map) elif tf_inspect.isfunction(o): node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) conversion_map.add_to_cache(o, node) if conversion_map.recursive: while True: candidate = None for obj in conversion_map.name_map.keys(): if obj not in conversion_map.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr(candidate, 'im_class') not in conversion_map.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, conversion_map, {}, {}) return node, name, ns
def isnamedtuple(f): """Returns True if the argument is a namedtuple-like.""" if not (tf_inspect.isclass(f) and issubclass(f, tuple)): return False if not hasattr(f, '_fields'): return False fields = getattr(f, '_fields') if not isinstance(fields, tuple): return False if not all(isinstance(f, str) for f in fields): return False return True
def convert_entity_to_ast(o, program_ctx): """Compile a Python entity into equivalent TensorFlow. Args: o: A Python entity. program_ctx: A ProgramContext object. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ logging.log(1, 'Converting %s', o) if tf_inspect.isclass(o): nodes, name, entity_info = convert_class_to_ast(o, program_ctx) elif tf_inspect.isfunction(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) elif tf_inspect.ismethod(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) # TODO(mdan,yashkatariya): Remove when object conversion is implemented. elif hasattr(o, '__class__'): raise NotImplementedError( 'Object conversion is not yet supported. If you are ' 'trying to convert code that uses an existing object, ' 'try including the creation of that object in the ' 'conversion. For example, instead of converting the method ' 'of a class, try converting the entire class instead. ' 'See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/' 'python/autograph/README.md#using-the-functional-api ' 'for more information.') else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) if logging.has_verbosity(2): logging.log(2, 'Compiled output of %s:\n\n%s\n', o, compiler.ast_to_source(nodes)) if logging.has_verbosity(4): for n in nodes: logging.log(4, 'Compiled AST of %s:\n\n%s\n\n', o, pretty_printer.fmt(n, color=False)) return nodes, name, entity_info
def object_to_graph(o, conversion_map, value_hints): """Compile a Python object into equivalent TensorFlow. The function will also recursively compile all the objects that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python object. conversion_map: A ConversionMap object. value_hints: A dict containing value hints for symbols like function parameters. Returns: A tuple (ast, new_name): * ast: An AST representing an object with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new object can be found. Raises: ValueError: if the object is not supported. """ if value_hints is None: value_hints = {} if tf_inspect.isclass(o): node, new_name = class_to_graph(o, conversion_map, value_hints) elif tf_inspect.isfunction(o): node, new_name = function_to_graph(o, conversion_map, value_hints) elif tf_inspect.ismethod(o): node, new_name = function_to_graph(o, conversion_map, value_hints) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) conversion_map.add_to_cache(o, node) if conversion_map.recursive: for obj in conversion_map.name_map.keys(): if obj not in conversion_map.dependency_cache: if (hasattr(obj, 'im_class') and getattr(obj, 'im_class') not in conversion_map.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue object_to_graph(obj, conversion_map, None) return node, new_name
def deserialize_keras_object(identifier, module_objects=None, custom_objects=None, printable_module_name='object'): if identifier is None: return None if isinstance(identifier, dict): # In this case we are dealing with a Keras config dictionary. config = identifier (cls, cls_config) = class_and_config_for_serialized_keras_object( config, module_objects, custom_objects, printable_module_name) if hasattr(cls, 'from_config'): arg_spec = tf_inspect.getfullargspec(cls.from_config) custom_objects = custom_objects or {} if 'custom_objects' in arg_spec.args: return cls.from_config( cls_config, custom_objects=dict( list(_GLOBAL_CUSTOM_OBJECTS.items()) + list(custom_objects.items()))) with CustomObjectScope(custom_objects): return cls.from_config(cls_config) else: # Then `cls` may be a function returning a class. # in this case by convention `config` holds # the kwargs of the function. custom_objects = custom_objects or {} with CustomObjectScope(custom_objects): return cls(**cls_config) elif isinstance(identifier, six.string_types): object_name = identifier if custom_objects and object_name in custom_objects: obj = custom_objects.get(object_name) elif object_name in _GLOBAL_CUSTOM_OBJECTS: obj = _GLOBAL_CUSTOM_OBJECTS[object_name] else: obj = module_objects.get(object_name) if obj is None: raise ValueError('Unknown ' + printable_module_name + ':' + object_name) # Classes passed by name are instantiated with no args, functions are # returned as-is. if tf_inspect.isclass(obj): return obj() return obj else: raise ValueError('Could not interpret serialized ' + printable_module_name + ': ' + identifier)
def generate_global_index(library_name, index, reference_resolver): """Given a dict of full names to python objects, generate an index page. The index page generated contains a list of links for all symbols in `index` that have their own documentation page. Args: library_name: The name for the documented library to use in the title. index: A dict mapping full names to python objects. reference_resolver: An instance of ReferenceResolver. Returns: A string containing an index page as Markdown. """ symbol_links = [] for full_name, py_object in six.iteritems(index): if (tf_inspect.ismodule(py_object) or tf_inspect.isfunction(py_object) or tf_inspect.isclass(py_object)): # In Python 3, unbound methods are functions, so eliminate those. if tf_inspect.isfunction(py_object): if full_name.count('.') == 0: parent_name = '' else: parent_name = full_name[:full_name.rfind('.')] if parent_name in index and tf_inspect.isclass(index[parent_name]): # Skip methods (=functions with class parents). continue symbol_links.append(( full_name, reference_resolver.python_link(full_name, full_name, '.'))) lines = ['# All symbols in %s' % library_name, ''] for _, link in sorted(symbol_links, key=lambda x: x[0]): lines.append('* %s' % link) # TODO(markdaoust): use a _ModulePageInfo -> prety_docs.build_md_page() return '\n'.join(lines)
def _is_free_function(py_object, full_name, index): """Check if input is a free function (and not a class- or static method).""" if not tf_inspect.isfunction(py_object): return False # Static methods are functions to tf_inspect (in 2.7), so check if the parent # is a class. If there is no parent, it's not a function. if '.' not in full_name: return False parent_name = full_name.rsplit('.', 1)[0] if tf_inspect.isclass(index[parent_name]): return False return True
def _score_name(self, name): """Return a tuple of scores indicating how to sort for the best name. This function is meant to be used as the `key` to the `sorted` function. This sorting in order: Prefers names refering to the defining class, over a subclass. Prefers names that are not in "contrib". prefers submodules to the root namespace. Prefers short names `tf.thing` over `tf.a.b.c.thing` Sorts lexicographically on name parts. Args: name: the full name to score, for example `tf.estimator.Estimator` Returns: A tuple of scores. When sorted the preferred name will have the lowest value. """ parts = name.split('.') short_name = parts[-1] container = self._index['.'.join(parts[:-1])] defining_class_score = 1 if tf_inspect.isclass(container): if short_name in container.__dict__: # prefer the defining class defining_class_score = -1 contrib_score = -1 if 'contrib' in parts: contrib_score = 1 while parts: parts.pop() container = self._index['.'.join(parts)] if tf_inspect.ismodule(container): break module_length = len(parts) if len(parts) == 2: # `tf.submodule.thing` is better than `tf.thing` module_length_score = -1 else: # shorter is better module_length_score = module_length return (defining_class_score, contrib_score, module_length_score, name)
def convert_entity_to_ast(o, program_ctx): """Compile a Python entity into equivalent TensorFlow. Args: o: A Python entity. program_ctx: A ProgramContext object. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ logging.log(1, 'Converting %s', o) if tf_inspect.isclass(o): nodes, name, entity_info = convert_class_to_ast(o, program_ctx) elif tf_inspect.isfunction(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) elif tf_inspect.ismethod(o): nodes, name, entity_info = convert_func_to_ast(o, program_ctx) elif hasattr(o, '__class__'): # Note: this should only be raised when attempting to convert the object # directly. converted_call should still support it. raise NotImplementedError( 'cannot convert entity "{}": object conversion is not yet' ' supported.'.format(o)) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) if logging.has_verbosity(2): logging.log(2, 'Compiled output of %s:\n\n%s\n', o, compiler.ast_to_source(nodes)) if logging.has_verbosity(4): for n in nodes: logging.log(4, 'Compiled AST of %s:\n\n%s\n\n', o, pretty_printer.fmt(n, color=False)) return nodes, name, entity_info
def _get_raw_docstring(py_object): """Get the docs for a given python object. Args: py_object: A python object to retrieve the docs for (class, function/method, or module). Returns: The docstring, or the empty string if no docstring was found. """ # For object instances, tf_inspect.getdoc does give us the docstring of their # type, which is not what we want. Only return the docstring if it is useful. if (tf_inspect.isclass(py_object) or tf_inspect.ismethod(py_object) or tf_inspect.isfunction(py_object) or tf_inspect.ismodule(py_object) or isinstance(py_object, property)): return tf_inspect.getdoc(py_object) or '' else: return ''
def _process_variable_assignment(self, source, targets): if isinstance(source, gast.Call): func = source.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): # This is then a constructor. anno.setanno(source, 'type', func_obj) anno.setanno(source, 'type_fqn', anno.getanno(func, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. for t in targets: if isinstance(t, gast.Tuple): for i, e in enumerate(t.elts): self.scope.setval(e.id, gast.Subscript( source, gast.Index(i), ctx=gast.Store())) else: self.scope.setval(t.id, source)
def _process_variable_assignment(self, target, value): # Constructors if isinstance(value, gast.Call): func = value.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): anno.setanno(value, 'is_constructor', True) anno.setanno(value, 'type', func_obj) anno.setanno(value, 'type_fqn', anno.getanno(func, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. if isinstance(target, (gast.Name, gast.Attribute)): target_symbol = anno.getanno(target, anno.Basic.QN) self.scope.setval(target_symbol, value) elif isinstance(target, gast.Subscript): pass else: raise ValueError('assignment target has unknown type: %s' % target)
def visit(unused_path, unused_parent, children): """Visitor that collects arguments for reordered functions.""" for child in children: _, attr = tf_decorator.unwrap(child[1]) api_names_v1 = tf_export.get_v1_names(attr) api_names_v1 = ['tf.%s' % name for name in api_names_v1] matches_function_names = any(name in function_names for name in api_names_v1) if matches_function_names: if tf_inspect.isclass(attr): # Get constructor arguments if attr is a class arg_list = tf_inspect.getargspec(getattr(attr, '__init__'))[0] arg_list = arg_list[1:] # skip 'self' argument else: # Get function arguments. # getargspec returns a tuple of (args, varargs, keywords, defaults) # we just look at args. arg_list = tf_inspect.getargspec(attr)[0] for name in api_names_v1: function_to_args[name] = arg_list
def wrapper(*args, **kwargs): """Wrapper that calls the compiled version of the wrapped function.""" partial_types = () arg_names = tf_inspect.getargspec(f)[0] for name, arg in zip(arg_names, args): arg_class = arg.__class__ if tf_inspect.isclass(arg_class): # If arg_value_hints specifies any name, use that instead. # TODO(mdan): Shouldn't this just be in the func's globals? if name not in arg_value_hints: arg_value_hints[name] = (arg_class.__name__, arg_class) # Annotated methods need to specify that their owner type is partial, # otherwise other members they call will not be converted. if name == 'self': partial_types = (arg_class,) wrapped = to_graph( f, recursive=recursive, arg_value_hints=arg_value_hints, partial_types=partial_types) return wrapped(*args, **kwargs)
def decorator(arg): """Registers a class with the Keras serialization framework.""" class_name = name if name is not None else arg.__name__ registered_name = package + '>' + class_name if tf_inspect.isclass(arg) and not hasattr(arg, 'get_config'): raise ValueError( 'Cannot register a class that does not have a get_config() method.') if registered_name in _GLOBAL_CUSTOM_OBJECTS: raise ValueError( '%s has already been registered to %s' % (registered_name, _GLOBAL_CUSTOM_OBJECTS[registered_name])) if arg in _GLOBAL_CUSTOM_NAMES: raise ValueError('%s has already been registered to %s' % (arg, _GLOBAL_CUSTOM_NAMES[arg])) _GLOBAL_CUSTOM_OBJECTS[registered_name] = arg _GLOBAL_CUSTOM_NAMES[arg] = registered_name return arg
def _process_variable_assignment(self, target, value): # Constructors if isinstance(value, gast.Call): func = value.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): anno.setanno(value, 'is_constructor', True) anno.setanno(value, 'type', func_obj) anno.setanno(value, 'type_fqn', anno.getanno(func, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. if isinstance(target, (gast.Name, gast.Attribute)): target_symbol = anno.getanno(target, anno.Basic.QN) self.scope.setval(target_symbol, value) elif isinstance(target, gast.Subscript): pass else: raise ValueError('assignment target has unknown type: %s' % target)
def visit(unused_path, unused_parent, children): """Visitor that collects arguments for reordered functions.""" for child in children: _, attr = tf_decorator.unwrap(child[1]) api_names_v1 = tf_export.get_v1_names(attr) api_names_v1 = ['tf.%s' % name for name in api_names_v1] matches_function_names = any( name in function_names for name in api_names_v1) if matches_function_names: if tf_inspect.isclass(attr): # Get constructor arguments if attr is a class arg_list = tf_inspect.getargspec( getattr(attr, '__init__'))[0] arg_list = arg_list[1:] # skip 'self' argument else: # Get function arguments. # getargspec returns a tuple of (args, varargs, keywords, defaults) # we just look at args. arg_list = tf_inspect.getargspec(attr)[0] for name in api_names_v1: function_to_args[name] = arg_list
def wrapper(*args, **kwargs): """Wrapper that calls the compiled version of the wrapped function.""" partial_types = () arg_values = {} arg_names = tf_inspect.getargspec(f)[0] for name, arg in zip(arg_names, args): arg_values[name] = arg arg_class = arg.__class__ # If arg_value_hints specifies any name, use that instead. if name not in arg_types: arg_types[name] = (arg_class.__name__, arg_class) if name == 'self' and tf_inspect.isclass(arg_class): # Annotated methods need to specify that their owner type is partial, # otherwise other members they call will not be converted. partial_types = (arg_class, ) wrapped = to_graph(f, recursive=recursive, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types) return wrapped(*args, **kwargs)
def collect_docs_for_module(self, parser_config): """Collect information necessary specifically for a module's doc page. Mainly this is information about the members of the module. Args: parser_config: An instance of ParserConfig. """ relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(self.full_name)) or '.') member_names = parser_config.tree.get(self.full_name, []) for name in member_names: if name in ['__builtins__', '__doc__', '__file__', '__name__', '__path__', '__package__']: continue member_full_name = self.full_name + '.' + name if self.full_name else name member = parser_config.py_name_to_object(member_full_name) member_doc = _parse_md_docstring(member, relative_path, parser_config.reference_resolver) url = parser_config.reference_resolver.reference_to_url( member_full_name, relative_path) if tf_inspect.ismodule(member): self._add_module(name, member_full_name, member, member_doc, url) elif tf_inspect.isclass(member): self._add_class(name, member_full_name, member, member_doc, url) elif tf_inspect.isfunction(member): self._add_function(name, member_full_name, member, member_doc, url) else: self._add_other_member(name, member_full_name, member, member_doc)
def _process_variable_assignment(self, source, targets): if isinstance(source, gast.Call): func = source.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): anno.setanno(source, 'is_constructor', True) anno.setanno(source, 'type', func_obj) anno.setanno(source, 'type_fqn', anno.getanno(func, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. for t in targets: if isinstance(t, gast.Tuple): # need to recurse on the case of assigning nested tuples, # ex. a, (b, c) = f() self._process_tuple_assignment(source, t) elif isinstance(t, (gast.Name, gast.Attribute)): self.scope.setval(anno.getanno(t, anno.Basic.QN), source) else: raise ValueError('Dont know how to handle assignment to %s' % t)
def _process_variable_assignment(self, source, targets): # Special case: constructors. if isinstance(source, gast.Call): func = source.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): anno.setanno(source, 'is_constructor', True) anno.setanno(source, 'type', func_obj) anno.setanno(source, 'type_fqn', anno.getanno(func, 'fqn')) # TODO (mdan): Raise an error if constructor has side effects. id:554 # https://github.com/imdone/tensorflow/issues/555 # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. # Multiple targets mean multiple assignment. for target in targets: # Tuple target means unpacking. if isinstance(target, (gast.Tuple, gast.List)): for i, target_item in enumerate(target.elts): # Two cases here: # 1. Static unpacking, e.g. a, b = c, d # 2. Dynamic unpacking, e.g. a, b = c # The former case is optimized away. if isinstance(source, (gast.Tuple, gast.List)): source_item = source.elts[i] else: source_item = gast.Subscript(source, gast.Index(i), ctx=None) self._process_variable_assignment(source_item, (target_item, )) elif isinstance(target, (gast.Name, gast.Attribute)): target_symbol = anno.getanno(target, anno.Basic.QN) self.scope.setval(target_symbol, source) else: raise ValueError('assignment target has unknown type: %s' % target)
def is_whitelisted_for_graph(o): """Check whether an entity is whitelisted for use in graph mode. Examples of whitelisted entities include all members of the tensorflow package. Args: o: A Python entity. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) for prefix, in config.DEFAULT_UNCOMPILED_MODULES: if m.__name__.startswith(prefix): return True if hasattr(o, 'autograph_info__'): return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if tf_inspect.isclass(o) and len(o.__bases__) > 1: logging.log_first_n( logging.level_warning(), 'Entity {} looks like a namedtuple subclass. If it has any custom' ' methods, they will not be converted by AutoGraph.'.format(o), 1) return True return False
def has_deprecation_decorator(symbol): """Checks if given object has a deprecation decorator. We check if deprecation decorator is in decorators as well as whether symbol is a class whose __init__ method has a deprecation decorator. Args: symbol: Python object. Returns: True if symbol has deprecation decorator. """ decorators, symbol = tf_decorator.unwrap(symbol) if contains_deprecation_decorator(decorators): return True if tf_inspect.isfunction(symbol): return False if not tf_inspect.isclass(symbol): return False if not hasattr(symbol, '__init__'): return False init_decorators, _ = tf_decorator.unwrap(symbol.__init__) return contains_deprecation_decorator(init_decorators)
def has_deprecation_decorator(symbol): """Checks if given object has a deprecation decorator. We check if deprecation decorator is in decorators as well as whether symbol is a class whose __init__ method has a deprecation decorator. Args: symbol: Python object. Returns: True if symbol has deprecation decorator. """ decorators, symbol = tf_decorator.unwrap(symbol) if contains_deprecation_decorator(decorators): return True if tf_inspect.isfunction(symbol): return False if not tf_inspect.isclass(symbol): return False if not hasattr(symbol, '__init__'): return False init_decorators, _ = tf_decorator.unwrap(symbol.__init__) return contains_deprecation_decorator(init_decorators)
def has_deprecation_decorator(symbol, decorators): """Checks if given object has a deprecation decorator. We check if deprecation decorator is in decorators as well as whether symbol is a class whose __init__ method has a deprecation decorator. Args: symbol: Unwrapped (i.e. without decorators) Python object. decorators: Decorators originally wrapped around symbol. Returns: True if symbol has deprecation decorator. """ if contains_deprecation_decorator(decorators): return True if tf_inspect.isfunction(symbol): return False if not tf_inspect.isclass(symbol): return False if not hasattr(symbol, '__init__'): return False init_decorators, _ = tf_decorator.unwrap(symbol.__init__) return contains_deprecation_decorator(init_decorators)
def _process_variable_assignment(self, source, targets): if isinstance(source, gast.Call): func = source.func if anno.hasanno(func, 'live_val'): func_obj = anno.getanno(func, 'live_val') if tf_inspect.isclass(func_obj): anno.setanno(source, 'is_constructor', True) anno.setanno(source, 'type', func_obj) anno.setanno(source, 'type_fqn', anno.getanno(func, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. for t in targets: if isinstance(t, gast.Tuple): # need to recurse on the case of assigning nested tuples, # ex. a, (b, c) = f() self._process_tuple_assignment(source, t) elif isinstance(t, (gast.Name, gast.Attribute)): self.scope.setval(anno.getanno(t, anno.Basic.QN), source) else: raise ValueError('Dont know how to handle assignment to %s' % t)
def wrapper(*args, **kwargs): """Wrapper that calls the compiled version of the wrapped function.""" partial_types = () arg_values = {} arg_names = tf_inspect.getargspec(f)[0] for name, arg in zip(arg_names, args): arg_values[name] = arg arg_class = arg.__class__ # If arg_value_hints specifies any name, use that instead. if name not in arg_types: arg_types[name] = (arg_class.__name__, arg_class) if name == 'self' and tf_inspect.isclass(arg_class): # Annotated methods need to specify that their owner type is partial, # otherwise other members they call will not be converted. partial_types = (arg_class,) wrapped = to_graph( f, recursive=recursive, verbose=verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types) return wrapped(*args, **kwargs)
def is_whitelisted_for_graph(o): """Check whether an entity is whitelisted for use in graph mode. Examples of whitelisted entities include all members of the tensorflow package. Args: o: A Python entity. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) for prefix, in config.DEFAULT_UNCOMPILED_MODULES: if m.__name__.startswith(prefix): return True if hasattr(o, 'autograph_info__'): return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if tf_inspect.isclass(o) and len(o.__bases__) > 1: logging.log_first_n( logging.level_warning(), 'Entity {} looks like a namedtuple subclass. If it has any custom' ' methods, they will not be converted by AutoGraph.'.format(o), 1) return True return False
def visit_Assign(self, node): self.generic_visit(node) if isinstance(node.value, gast.Call): target = node.value.func if anno.hasanno(target, 'live_val'): target_obj = anno.getanno(target, 'live_val') if tf_inspect.isclass(target_obj): # This is then a constructor. anno.setanno(node.value, 'type', target_obj) anno.setanno(node.value, 'type_fqn', anno.getanno(target, 'fqn')) # TODO(mdan): Raise an error if constructor has side effects. # We can have a whitelist of no-side-effects constructors. # We can also step inside the constructor and further analyze. for n in node.targets: if isinstance(n, gast.Tuple): for i, e in enumerate(n.elts): self.scope.setval(e.id, gast.Subscript( node.value, gast.Index(i), ctx=gast.Store())) else: self.scope.setval(n.id, node.value) return node
def entity_to_graph(o, program_ctx, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. program_ctx: A ProgramContext object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, program_ctx) elif tf_inspect.isfunction(o): # TODO(mdan): This is not a reliable mechanism. # The most reliable way is to check the source code, the AST will contain # a Lambda node instead of a FunctionDef if o.__name__ == '<lambda>': raise NotImplementedError( 'lambda functions are not yet supported; declare the function' ' using def instead: %s' % o) else: node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) program_ctx.add_to_cache(o, node) if program_ctx.recursive: while True: candidate = None for obj in program_ctx.name_map.keys(): if obj not in program_ctx.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr( candidate, 'im_class') not in program_ctx.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, program_ctx, {}, {}) return node, name, ns
def converted_call(f, owner, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" logging.vlog(logging.DEBUG, 'Converted call: %s; owner: %s', f, owner) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): # Args typically include `self`, as required by the conversion process. # When conversion is skipped, `self` is not necessary, because the # original bound method is being executed. This code removes it. if tf_inspect.ismethod(f) and args: f_class = inspect_utils.getmethodclass(f) if args[0] is f_class: args = args[1:] return f(*args, **kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return f(*args, **kwargs) # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_class = inspect_utils.getmethodclass(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_class is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner, ) + args else: # When the owner is not specified, use the result of # inspect_utils.getmethodclass. # TODO(b/119246461): Make sure an owner is always specified. if not args or args[0] is not f_class: effective_args = (f_class, ) + args else: effective_args = (f_class, ) + args[1:] partial_types = (f_class, ) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f, ) + args partial_types = (f.__class__, ) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__, ) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'], ) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=arg_values, arg_types=arg_types, experimental_optional_features=options.optional_features, experimental_strip_decorators=options.strip_decorators, experimental_verbose=options.verbose, experimental_partial_types=partial_types) result = converted_f(*effective_args, **kwargs) # The converted function's closure is simply inserted into the function's # module __dict__. Since modules are permanently cached, that results in # leaking the entire closure. # Normally, it's not safe to delete the module because that may release said # closure as well. However, in the case of converted_call we are certain the # function will not be executed again, so the closure should no longer be # needed so long as the function doesn't return any executable code. # TODO(mdan): Attach the closure properly, using cells. if all(map(_is_not_callable, nest.flatten(result))): del sys.modules[converted_f.__module__] return result
def __call__(self, path, parent, children): # The path to the object. lib_path = 'tensorflow.%s' % path if path else 'tensorflow' _, parent = tf_decorator.unwrap(parent) # A small helper method to construct members(children) protos. def _AddMember(member_name, member_obj, proto): """Add the child object to the object being constructed.""" _, member_obj = tf_decorator.unwrap(member_obj) if (_SkipMember(parent, member_name) or isinstance( member_obj, deprecation.HiddenTfApiAttribute)): return if member_name == '__init__' or not member_name.startswith('_'): if tf_inspect.isroutine(member_obj): new_method = proto.member_method.add() new_method.name = member_name # If member_obj is a python builtin, there is no way to get its # argspec, because it is implemented on the C side. It also has no # func_code. if hasattr(member_obj, '__code__'): new_method.argspec = _SanitizedArgSpec(member_obj) else: new_member = proto.member.add() new_member.name = member_name if tf_inspect.ismodule(member_obj): new_member.mtype = "<type \'module\'>" else: new_member.mtype = _NormalizeType(str( type(member_obj))) parent_corner_cases = _CORNER_CASES.get(path, {}) if path not in _CORNER_CASES or parent_corner_cases: # Decide if we have a module or a class. if tf_inspect.ismodule(parent): # Create a module object. module_obj = api_objects_pb2.TFAPIModule() for name, child in children: if name in parent_corner_cases: # If we have an empty entry, skip this object. if parent_corner_cases[name]: module_obj.member.add( **(parent_corner_cases[name])) else: _AddMember(name, child, module_obj) # Store the constructed module object. self._protos[lib_path] = api_objects_pb2.TFAPIObject( path=lib_path, tf_module=module_obj) elif _IsProtoClass(parent): proto_obj = api_objects_pb2.TFAPIProto() parent.DESCRIPTOR.CopyToProto(proto_obj.descriptor) # Store the constructed proto object. self._protos[lib_path] = api_objects_pb2.TFAPIObject( path=lib_path, tf_proto=proto_obj) elif tf_inspect.isclass(parent): # Construct a class. class_obj = api_objects_pb2.TFAPIClass() class_obj.is_instance.extend( _NormalizeIsInstance(i) for i in _SanitizedMRO(parent)) for name, child in children: if name in parent_corner_cases: # If we have an empty entry, skip this object. if parent_corner_cases[name]: class_obj.member.add(**(parent_corner_cases[name])) else: _AddMember(name, child, class_obj) # Store the constructed class object. self._protos[lib_path] = api_objects_pb2.TFAPIObject( path=lib_path, tf_class=class_obj) else: logging.error( 'Illegal call to ApiProtoDump::_py_obj_to_proto.' 'Object is neither a module nor a class: %s', path)
def getmethodclass(m): """Resolves a function's owner, e.g. a method's class. Note that this returns the object that the function was retrieved from, not necessarily the class where it was defined. This function relies on Python stack frame support in the interpreter, and has the same limitations that inspect.currentframe. Limitations. This function will only work correctly if the owned class is visible in the caller's global or local variables. Args: m: A user defined function Returns: The class that this function was retrieved from, or None if the function is not an object or class method, or the class that owns the object or method is not visible to m. Raises: ValueError: if the class could not be resolved for any unexpected reason. """ # Callable objects: return their own class. if (not hasattr(m, '__name__') and hasattr(m, '__class__') and hasattr(m, '__call__')): if isinstance(m.__class__, six.class_types): return m.__class__ # Instance method and class methods: should be bound to a non-null "self". # If self is a class, then it's a class method. if hasattr(m, '__self__'): if m.__self__: if tf_inspect.isclass(m.__self__): return m.__self__ return type(m.__self__) # Class, static and unbound methods: search all defined classes in any # namespace. This is inefficient but more robust method. owners = [] caller_frame = tf_inspect.currentframe().f_back try: # TODO(mdan): This doesn't consider cell variables. # TODO(mdan): This won't work if the owner is hidden inside a container. # Cell variables may be pulled using co_freevars and the closure. for v in itertools.chain(caller_frame.f_locals.values(), caller_frame.f_globals.values()): if hasattr(v, m.__name__): candidate = getattr(v, m.__name__) # Py2 methods may be bound or unbound, extract im_func to get the # underlying function. if hasattr(candidate, 'im_func'): candidate = candidate.im_func if hasattr(m, 'im_func'): m = m.im_func if candidate is m: owners.append(v) finally: del caller_frame if owners: if len(owners) == 1: return owners[0] # If multiple owners are found, and are not subclasses, raise an error. owner_types = tuple(o if tf_inspect.isclass(o) else type(o) for o in owners) for o in owner_types: if tf_inspect.isclass(o) and issubclass(o, tuple(owner_types)): return o raise ValueError('Found too many owners of %s: %s' % (m, owners)) return None
def SmartSet(self, obj, attr_name, new_attr): """Replace obj.attr_name with new_attr. This method is smart and works at the module, class, and instance level while preserving proper inheritance. It will not stub out C types however unless that has been explicitly allowed by the type. This method supports the case where attr_name is a staticmethod or a classmethod of obj. Notes: - If obj is an instance, then it is its class that will actually be stubbed. Note that the method Set() does not do that: if obj is an instance, it (and not its class) will be stubbed. - The stubbing is using the builtin getattr and setattr. So, the __get__ and __set__ will be called when stubbing (TODO: A better idea would probably be to manipulate obj.__dict__ instead of getattr() and setattr()). Args: obj: The object whose attributes we want to modify. attr_name: The name of the attribute to modify. new_attr: The new value for the attribute. Raises: AttributeError: If the attribute cannot be found. """ _, obj = tf_decorator.unwrap(obj) if (tf_inspect.ismodule(obj) or (not tf_inspect.isclass(obj) and attr_name in obj.__dict__)): orig_obj = obj orig_attr = getattr(obj, attr_name) else: if not tf_inspect.isclass(obj): mro = list(tf_inspect.getmro(obj.__class__)) else: mro = list(tf_inspect.getmro(obj)) mro.reverse() orig_attr = None found_attr = False for cls in mro: try: orig_obj = cls orig_attr = getattr(obj, attr_name) found_attr = True except AttributeError: continue if not found_attr: raise AttributeError('Attribute not found.') # Calling getattr() on a staticmethod transforms it to a 'normal' function. # We need to ensure that we put it back as a staticmethod. old_attribute = obj.__dict__.get(attr_name) if old_attribute is not None and isinstance(old_attribute, staticmethod): orig_attr = staticmethod(orig_attr) self.stubs.append((orig_obj, attr_name, orig_attr)) setattr(orig_obj, attr_name, new_attr)
def converted_call(f, recursive, verbose, arg_types, *args, **kwargs): """Compiles a function call inline.""" # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) unknown_arg_value = object() # Sentinel for arguments of unknown value if inspect_utils.isbuiltin(f): return builtins.dynamic_builtin(f, *args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f effective_args = args f_class = inspect_utils.getmethodclass(f) if f_class is not None: partial_types = (f_class,) else: partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f,) + args partial_types = (f.__class__,) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) for name, arg in arg_values.items(): if arg is unknown_arg_value: continue arg_class = arg.__class__ # If arg_value_hints specifies any name, use that instead. if name not in arg_types: arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__,) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'],) converted_f = to_graph( target_entity, recursive=recursive, verbose=verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types) return converted_f(*effective_args, **kwargs)
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow', search_hints=True, site_api_path=None): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_api_path: Used to write the api-duplicates _redirects.yaml file. if None (the default) the file is not generated. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError("'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) if not os.path.exists(output_dir): os.makedirs(output_dir) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Collect redirects for an api _redirects.yaml file. redirects = ['redirects:\n'] # Parse and write Markdown pages, resolving cross-links (@{symbol}). for full_name, py_object in six.iteritems(parser_config.index): parser_config.reference_resolver.current_doc_full_name = full_name if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or _is_free_function( py_object, full_name, parser_config.index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_children.setdefault(subname, []).append(full_name) break # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) # This function returns raw bytes in PY2 or unicode in PY3. if search_hints: content = [page_info.get_metadata_html()] else: content = [''] content.append(pretty_docs.build_md_page(page_info)) text = '\n'.join(content) if six.PY3: text = text.encode('utf-8') with open(path, 'wb') as f: f.write(text) except OSError: raise OSError('Cannot write documentation for %s to %s' % (full_name, directory)) if site_api_path: duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] template = ('- from: /{}\n' ' to: /{}\n') for dup in duplicates: from_path = os.path.join(site_api_path, dup.replace('.', '/')) to_path = os.path.join(site_api_path, full_name.replace('.', '/')) redirects.append(template.format(from_path, to_path)) if site_api_path: api_redirects_path = os.path.join(output_dir, '_redirects.yaml') with open(api_redirects_path, 'w') as redirect_file: redirect_file.write(''.join(redirects)) if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write( '# Automatically generated file; please do not edit\ntoc:\n') for module in modules: indent_num = module.count('.') # Don't list `tf.submodule` inside `tf` indent_num = max(indent_num, 1) indent = ' ' * indent_num if indent_num > 1: # tf.contrib.baysflow.entropy will be under # tf.contrib->baysflow->entropy title = module.split('.')[-1] else: title = module header = [ '- title: ' + title, ' section:', ' - title: Overview', ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[module] ] header = ''.join([indent + line + '\n' for line in header]) f.write(header) symbols_in_module = module_children.get(module, []) # Sort case-insensitive, if equal sort case sensitive (upper first) symbols_in_module.sort(key=lambda a: (a.upper(), a)) for full_name in symbols_in_module: item = [ ' - title: ' + full_name[len(module) + 1:], ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[full_name] ] item = ''.join([indent + line + '\n' for line in item]) f.write(item) # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index(root_title, parser_config.index, parser_config.reference_resolver))
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) owner_attr = f # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if logging.has_verbosity(1): if owner is not None: composite_desc = '("{}" attr of {})'.format(owner_attr, owner) else: composite_desc = '' logging.log(1, 'Converted call: %s %s\n args: %s\n kwargs: %s\n', f, composite_desc, args, kwargs) if inspect_utils.isbuiltin(f): if kwargs: return py_builtins.overload_of(f)(*args, **kwargs) else: return py_builtins.overload_of(f)(*args) # TODO(mdan): Clean up the naming inconsistency. if hasattr(f, 'autograph_info__') or hasattr(f, '__ag_compiled'): logging.log(2, 'Permanently whitelisted: %s: already converted', f) return _call_unconverted(f, args, kwargs) # TODO(b/122265385): Remove this bypass. if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f)) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return _call_unconverted(f, args, kwargs) if _is_known_loaded_type(f, 'functools', '_lru_cache_wrapper'): logging.log(2, 'Permanently whitelisted: %s: lru_cache', f) return _call_unconverted(f, args, kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return _call_unconverted(f, args, kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. if any(f in m.__dict__.values() for m in (collections, pdb, copy, inspect, re)): logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return _call_unconverted(f, args, kwargs) # Custom ops and kernels are also permanently whitelisted. # See tensorflow.framework.load_library. if (hasattr(f, '__module__') and hasattr(f.__module__, '_IS_TENSORFLOW_PLUGIN')): logging.log(2, 'Permanently whitelisted: %s: TensorFlow plugin', f) return _call_unconverted(f, args, kwargs) if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return _call_unconverted(f, args, kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return _call_unconverted(f, args, kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) if kwargs is not None: new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: effective_args = (f_self, ) + args else: effective_args = args elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f effective_args = args elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ effective_args = (f, ) + args else: target_entity = f raise NotImplementedError('unknown callable type "%s"' % type(f)) if not tf_inspect.isclass(target_entity): if not hasattr(target_entity, '__code__'): logging.log(2, 'Permanently whitelisted: %s: native binding', target_entity) return _call_unconverted(f, args, kwargs) elif (hasattr(target_entity.__code__, 'co_filename') and target_entity.__code__.co_filename == '<string>'): # TODO(mdan): __globals__['txt'] might work in Py3. logging.log( 2, 'Permanently whitelisted: %s: dynamic code (exec?)', target_entity) return _call_unconverted(f, args, kwargs) converted_f = to_graph( target_entity, recursive=options.recursive, experimental_optional_features=options.optional_features) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) if six.PY3: logging.log(2, 'KW defaults of %s : %s', converted_f, converted_f.__kwdefaults__) if kwargs is not None: callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) else: callargs = tf_inspect.getcallargs(converted_f, *effective_args) formatted_callargs = '\n'.join(' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) except Exception as e: # pylint:disable=broad-except logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) if is_autograph_strict_conversion_mode(): raise logging.warn( 'Entity %s could not be transformed and will be executed as-is.' ' Please report this to the AutoGraph team. When filing the bug, set' ' the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and' ' attach the full output. Cause: %s', target_entity, e) return _call_unconverted(f, args, kwargs) with StackTraceMapper(converted_f), tf_stack.CurrentModuleFilter(): try: if kwargs is not None: result = converted_f(*effective_args, **kwargs) else: result = converted_f(*effective_args) except Exception as e: _attach_metadata(e, converted_f, True) raise return result
def is_whitelisted_for_graph(o): """Check whether an entity is whitelisted for use in graph mode. Examples of whitelisted entities include all members of the tensorflow package. Args: o: A Python entity. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) if not hasattr(m, '__name__'): # Note: typically it's builtins that fall in this category. Builtins will # be handled by specific code that follows this screening layer. logging.log(2, '%s is NOT whitelisted: unknown module name', o) return False for prefix, in config.DEFAULT_UNCOMPILED_MODULES: if m.__name__.startswith(prefix): logging.log(2, '%s is whitelisted: name starts with "%s"', o, prefix) return True if hasattr(o, 'autograph_info__') or hasattr(o, '__ag_compiled'): logging.log(2, '%s is whitelisted: already converted', o) return True if (not inspect_utils.isweakrefself(o) and not tf_inspect.isclass(o) and hasattr(o, '__call__') and hasattr(o, '__class__')): # Callable objects: whitelisted if their __call__ method is. call_whitelisted = is_whitelisted_for_graph(o.__call__) if call_whitelisted: logging.log(2, '%s is whitelisted: object __call__ whitelisted', o) return call_whitelisted if tf_inspect.ismethod(o): # Methods of whitelisted classes are also whitelisted, even if they are # bound via user subclasses. # # For example, suppose `tf.Foo` has a method called `bar`, and `baz` is # defined as below. `tf.Foo` is whitelisted. Then `baz.bar` is also # whitelisted. # # class Custom(tf.Foo): # pass # # baz = Custom() # # For the example above, if `Custom` did overload `bar`, then it would no # longer be whitelisted. owner_class = inspect_utils.getmethodclass(o) if owner_class is not None: owner_class = inspect_utils.getdefiningclass(o, owner_class) if is_whitelisted_for_graph(owner_class): logging.log(2, '%s is whitelisted: owner is whitelisted %s', o, owner_class) return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if tf_inspect.isclass(o) and len(o.__bases__) > 1: logging.warn_first_n( 'Entity {} looks like a namedtuple subclass. If it has any custom' ' methods, they will not be converted by AutoGraph.'.format(o), 1) logging.log(2, '%s is whitelisted: named tuple', o) return True logging.log(2, '%s is NOT whitelisted', o) return False
def deprecated_alias(deprecated_name, name, func_or_class, warn_once=True): """Deprecate a symbol in favor of a new name with identical semantics. This function is meant to be used when defining a backwards-compatibility alias for a symbol which has been moved. For example: module1.py: ```python class NewNameForClass: pass ``` module2.py: ```python import module1 DeprecatedNameForClass = deprecated_alias( deprecated_name='module2.DeprecatedNameForClass', name='module1.NewNameForClass', func_or_class=module1.NewNameForClass) ``` This function works for classes and functions. For classes, it creates a new class which is functionally identical (it inherits from the original, and overrides its constructor), but which prints a deprecation warning when an instance is created. It also adds a deprecation notice to the class' docstring. For functions, it returns a function wrapped by `tf_decorator.make_decorator`. That function prints a warning when used, and has a deprecation notice in its docstring. This is more or less equivalent (the deprecation warning has slightly different text) to writing: ```python @deprecated def deprecated_alias(original_args): real_function(original_args) ``` Args: deprecated_name: The name of the symbol that is being deprecated, to be used in the warning message. This should be its fully qualified name to avoid confusion. name: The name of the symbol that is to be used instead of the deprecated name. This should be a fully qualified name to avoid confusion. func_or_class: The (non-deprecated) class or function for which a deprecated alias should be created. warn_once: If True (the default), only print a deprecation warning the first time this function is used, or the class is instantiated. Returns: A wrapped version of `func_or_class` which prints a deprecation warning on use and has a modified docstring. """ if tf_inspect.isclass(func_or_class): # Make a new class with __init__ wrapped in a warning. class _NewClass(func_or_class): # pylint: disable=missing-docstring __doc__ = decorator_utils.add_notice_to_docstring( func_or_class.__doc__, 'Please use %s instead.' % name, 'DEPRECATED CLASS', '(deprecated)', [ 'THIS CLASS IS DEPRECATED. ' 'It will be removed in a future version. ' ]) __name__ = func_or_class.__name__ __module__ = _call_location(outer=True) @_wrap_decorator(func_or_class.__init__) def __init__(self, *args, **kwargs): if hasattr(_NewClass.__init__, '__func__'): # Python 2 _NewClass.__init__.__func__.__doc__ = func_or_class.__init__.__doc__ else: # Python 3 _NewClass.__init__.__doc__ = func_or_class.__init__.__doc__ if _PRINT_DEPRECATION_WARNINGS: # We're making the alias as we speak. The original may have other # aliases, so we cannot use it to check for whether it's already been # warned about. if _NewClass.__init__ not in _PRINTED_WARNING: if warn_once: _PRINTED_WARNING[_NewClass.__init__] = True logging.warning( 'From %s: The name %s is deprecated. Please use %s instead.\n', _call_location(), deprecated_name, name) super(_NewClass, self).__init__(*args, **kwargs) return _NewClass else: decorator_utils.validate_callable(func_or_class, 'deprecated') # Make a wrapper for the original @functools.wraps(func_or_class) def new_func(*args, **kwargs): # pylint: disable=missing-docstring if _PRINT_DEPRECATION_WARNINGS: # We're making the alias as we speak. The original may have other # aliases, so we cannot use it to check for whether it's already been # warned about. if new_func not in _PRINTED_WARNING: if warn_once: _PRINTED_WARNING[new_func] = True logging.warning( 'From %s: The name %s is deprecated. Please use %s instead.\n', _call_location(), deprecated_name, name) return func_or_class(*args, **kwargs) return tf_decorator.make_decorator( func_or_class, new_func, 'deprecated', _add_deprecated_function_notice_to_docstring( func_or_class.__doc__, None, 'Please use %s instead.' % name))
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" logging.log( 1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f, owner, args, kwargs) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): if kwargs: return py_builtins.overload_of(f)(*args, **kwargs) else: return py_builtins.overload_of(f)(*args) if _is_known_loaded_type(f, 'weakref', 'ref'): logging.log(2, 'Permanently whitelisted: %s: weakref', f) return _call_unconverted(f, args, kwargs) # TODO(b/122265385): Remove this bypass. if (_is_known_loaded_type(f, 'wrapt', 'FunctionWrapper') or _is_known_loaded_type(f, 'wrapt', 'BoundFunctionWrapper')): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f)) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return _call_unconverted(f, args, kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return _call_unconverted(f, args, kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. # Note: TF linter disallows importing inspect. if any(f in m.__dict__.values() for m in (collections, pdb, copy, tf_inspect._inspect)): # pylint:disable=protected-access logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return _call_unconverted(f, args, kwargs) if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return _call_unconverted(f, args, kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return _call_unconverted(f, args, kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) if kwargs is not None: new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: effective_args = (f_self, ) + args else: effective_args = args elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f effective_args = args elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ effective_args = (f, ) + args else: target_entity = f raise NotImplementedError('unknown callable type "%s"' % type(f)) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=None, arg_types=None, experimental_optional_features=options.optional_features) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) if kwargs is not None: callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) else: callargs = tf_inspect.getcallargs(converted_f, *effective_args) formatted_callargs = '\n'.join(' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) # TODO(mdan): Reduce this list. except (errors.AutoGraphError, AssertionError, AttributeError, IndexError, KeyError, NameError, NotImplementedError, SyntaxError, TypeError, ValueError, IOError) as e: logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) if is_autograph_strict_conversion_mode(): raise logging.warn( 'Entity %s could not be transformed and will be executed as-is.' ' Some features (e.g. tensor-dependent conditionals and loops) may not' ' work as expected.' ' Error details can be found in the logs when running with the env' ' variable AUTOGRAPH_VERBOSITY >= 1. Please report this to the' ' AutoGraph team. Cause: %s', target_entity, e) return _call_unconverted(f, args, kwargs) if kwargs is not None: result = converted_f(*effective_args, **kwargs) else: result = converted_f(*effective_args) return result
def make_variable(name, shape=None, dtype=dtypes.float32, initializer=None, trainable=None, caching_device=None, validate_shape=True, constraint=None, use_resource=None, collections=None, synchronization=tf_variables.VariableSynchronization.AUTO, aggregation=tf_variables.VariableAggregation.NONE, partitioner=None): # pylint: disable=unused-argument """Temporary util to create a variable (relies on `variable_scope.variable`). Some reuse-related technicalities prevent us from using `variable_scope.get_variable()` directly, so we use a subcomponent that has fewer constraints (`variable_scope.variable()`). In the longer term, it seems like a similar "default variable creator" method should exist in `Trackable` instead. When this happens, we can get rid of this temporary solution. TODO(fchollet): remove this method when no longer needed. Arguments: name: Variable name. shape: Variable shape. dtype: The type of the variable. Defaults to `self.dtype` or `float32`. initializer: Initializer instance (callable). trainable: Whether the variable should be part of the layer's "trainable_variables" (e.g. variables, biases) or "non_trainable_variables" (e.g. BatchNorm mean, stddev). Note, if the current variable scope is marked as non-trainable then this parameter is ignored and any added variables are also marked as non-trainable. `trainable` defaults to `True` unless `synchronization` is set to `ON_READ`. caching_device: Passed to `tf.Variable`. validate_shape: Passed to `tf.Variable`. constraint: Constraint instance (callable). use_resource: Whether to use a `ResourceVariable`. collections: List of graph collections keys. The new variable is added to these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`. synchronization: Indicates when a distributed a variable will be aggregated. Accepted values are constants defined in the class `tf.VariableSynchronization`. By default the synchronization is set to `AUTO` and the current `DistributionStrategy` chooses when to synchronize. If `synchronization` is set to `ON_READ`, `trainable` must not be set to `True`. aggregation: Indicates how a distributed variable will be aggregated. Accepted values are constants defined in the class `tf.VariableAggregation`. partitioner: Not handled at this time. Returns: Variable instance. """ initializing_from_value = False if initializer is not None and not callable(initializer): initializing_from_value = True if initializing_from_value: init_val = initializer variable_dtype = None else: # Instantiate initializer if provided initializer is a type object. if tf_inspect.isclass(initializer): initializer = initializer() init_val = functools.partial(initializer, shape, dtype=dtype) variable_dtype = dtype.base_dtype if use_resource is None: use_resource = True # TODO(apassos,rohanj) figure out how to remove collections from here so we # can remove the V1. variable_shape = tensor_shape.TensorShape(shape) return tf_variables.VariableV1( initial_value=init_val, name=name, trainable=trainable, caching_device=caching_device, dtype=variable_dtype, validate_shape=validate_shape, constraint=constraint, use_resource=use_resource, collections=collections, synchronization=synchronization, aggregation=aggregation, shape=variable_shape if variable_shape else None)
def populate_deserializable_objects(): """Populates dict ALL_OBJECTS with every built-in layer. """ global LOCAL if not hasattr(LOCAL, 'ALL_OBJECTS'): LOCAL.ALL_OBJECTS = {} LOCAL.GENERATED_WITH_V2 = None if LOCAL.ALL_OBJECTS and LOCAL.GENERATED_WITH_V2 == tf2.enabled(): # Objects dict is already generated for the proper TF version: # do nothing. return LOCAL.ALL_OBJECTS = {} LOCAL.GENERATED_WITH_V2 = tf2.enabled() base_cls = base_layer.Layer generic_utils.populate_dict_with_module_objects( LOCAL.ALL_OBJECTS, ALL_MODULES, obj_filter=lambda x: inspect.isclass(x) and issubclass(x, base_cls)) # Overwrite certain V1 objects with V2 versions if tf2.enabled(): generic_utils.populate_dict_with_module_objects( LOCAL.ALL_OBJECTS, ALL_V2_MODULES, obj_filter=lambda x: inspect.isclass(x) and issubclass( x, base_cls)) # These deserialization aliases are added for backward compatibility, # as in TF 1.13, "BatchNormalizationV1" and "BatchNormalizationV2" # were used as class name for v1 and v2 version of BatchNormalization, # respectively. Here we explicitly convert them to their canonical names. LOCAL.ALL_OBJECTS[ 'BatchNormalizationV1'] = normalization.BatchNormalization LOCAL.ALL_OBJECTS[ 'BatchNormalizationV2'] = normalization_v2.BatchNormalization # Prevent circular dependencies. from tensorflow.python.keras import models # pylint: disable=g-import-not-at-top from tensorflow.python.keras.premade.linear import LinearModel # pylint: disable=g-import-not-at-top from tensorflow.python.keras.premade.wide_deep import WideDeepModel # pylint: disable=g-import-not-at-top from tensorflow.python.keras.feature_column.sequence_feature_column import SequenceFeatures # pylint: disable=g-import-not-at-top LOCAL.ALL_OBJECTS['Input'] = input_layer.Input LOCAL.ALL_OBJECTS['InputSpec'] = input_spec.InputSpec LOCAL.ALL_OBJECTS['Network'] = models.Network LOCAL.ALL_OBJECTS['Model'] = models.Model LOCAL.ALL_OBJECTS['SequenceFeatures'] = SequenceFeatures LOCAL.ALL_OBJECTS['Sequential'] = models.Sequential LOCAL.ALL_OBJECTS['LinearModel'] = LinearModel LOCAL.ALL_OBJECTS['WideDeepModel'] = WideDeepModel if tf2.enabled(): LOCAL.ALL_OBJECTS.update(FEATURE_COLUMN_V2_OBJECTS) else: LOCAL.ALL_OBJECTS.update(FEATURE_COLUMN_V1_OBJECTS) # Merge layers, function versions. LOCAL.ALL_OBJECTS['add'] = merge.add LOCAL.ALL_OBJECTS['subtract'] = merge.subtract LOCAL.ALL_OBJECTS['multiply'] = merge.multiply LOCAL.ALL_OBJECTS['average'] = merge.average LOCAL.ALL_OBJECTS['maximum'] = merge.maximum LOCAL.ALL_OBJECTS['minimum'] = merge.minimum LOCAL.ALL_OBJECTS['concatenate'] = merge.concatenate LOCAL.ALL_OBJECTS['dot'] = merge.dot
def is_whitelisted_for_graph(o, check_call_override=True): """Checks whether an entity is whitelisted for use in graph mode. Examples of whitelisted entities include all members of the tensorflow package. Args: o: A Python entity. check_call_override: Reserved for internal use. When set to `False`, it disables the rule according to which classes are whitelisted if their __call__ method is whitelisted. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) if hasattr(m, '__name__'): # Builtins typically have unnamed modules. for prefix, in config.DEFAULT_UNCOMPILED_MODULES: if m.__name__.startswith(prefix + '.') or m.__name__ == prefix: logging.log(2, 'Whitelisted: %s: name starts with "%s"', o, prefix) return True if hasattr(o, 'autograph_info__') or hasattr(o, '__ag_compiled'): logging.log(2, 'Whitelisted: %s: already converted', o) return True if tf_inspect.isgeneratorfunction(o): logging.warn( 'Entity {} appears to be a generator function. It will not be converted' ' by AutoGraph.'.format(o), 1) logging.log(2, 'Whitelisted: %s: generator functions are not converted', o) return True if check_call_override and hasattr(o, '__call__'): # Callable objects: whitelisted if their __call__ method is. # The type check avoids infinite recursion around the __call__ method # of function objects. if (type(o) != type(o.__call__)) and is_whitelisted_for_graph( o.__call__): # pylint: disable=unidiomatic-typecheck logging.log(2, 'Whitelisted: %s: object __call__ whitelisted', o) return True owner_class = None if tf_inspect.ismethod(o): # Methods of whitelisted classes are also whitelisted, even if they are # bound via user subclasses. # # For example, suppose `tf.Foo` has a method called `bar`, and `baz` is # defined as below. `tf.Foo` is whitelisted. Then `baz.bar` is also # whitelisted. # # class Custom(tf.Foo): # pass # # baz = Custom() # # For the example above, if `Custom` did overload `bar`, then it would no # longer be whitelisted. owner_class = inspect_utils.getmethodclass(o) if owner_class is not None: if issubclass(owner_class, unittest.TestCase): logging.log(2, 'Whitelisted: %s: method of TestCase subclass', o) return True owner_class = inspect_utils.getdefiningclass(o, owner_class) is_call_override = (o.__name__ == '__call__') if is_whitelisted_for_graph( owner_class, check_call_override=not is_call_override): logging.log(2, 'Whitelisted: %s: owner is whitelisted %s', o, owner_class) return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if tf_inspect.isclass(o) and len(o.__bases__) > 1: logging.warn( 'Entity {} looks like a namedtuple subclass. Its constructor will' ' not be converted by AutoGraph, but if it has any custom methods,' ' those will be.'.format(o), 1) logging.log(2, 'Whitelisted: %s: named tuple', o) return True logging.log(2, 'Not whitelisted: %s: default rule', o) return False
def converted_call(f, owner, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) unknown_arg_value = object() # Sentinel for arguments of unknown value if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_class = inspect_utils.getmethodclass(f) if f_class is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner,) + args else: effective_args = args partial_types = (f_class,) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f,) + args partial_types = (f.__class__,) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): if arg is unknown_arg_value: continue arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__,) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'],) converted_f = to_graph( target_entity, recursive=options.recursive, verbose=options.verbose, arg_values=arg_values, arg_types=arg_types, partial_types=partial_types, strip_decorators=options.strip_decorators) return converted_f(*effective_args, **kwargs)
def entity_to_graph(o, program_ctx, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. program_ctx: A ProgramContext object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if program_ctx.options.verbose == converter.Verbosity.VERBOSE: logging.info('Converting {}'.format(o)) if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, program_ctx) elif tf_inspect.isfunction(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) # TODO(mdan,yashkatariya): Remove when object conversion is implemented. elif hasattr(o, '__class__'): raise NotImplementedError( 'Object conversion is not yet supported. If you are ' 'trying to convert code that uses an existing object, ' 'try including the creation of that object in the ' 'conversion. For example, instead of converting the method ' 'of a class, try converting the entire class instead. ' 'See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/' 'contrib/autograph/README.md#using-the-functional-api ' 'for more information.') else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) # TODO(mdan): This is temporary. it should be created using a converter. # TODO(mdan): The attribute should be added with a helper, not directly. # The helper can ensure there are no collisions. template = ''' entity.autograph_info__ = {} ''' node.extend(templates.replace(template, entity=name)) program_ctx.add_to_cache(o, node) if program_ctx.options.verbose == converter.Verbosity.VERBOSE: logging.info('Compiled output of {}:\n\n{}\n'.format( o, compiler.ast_to_source(node))) if program_ctx.options.recursive: while True: candidate = None for obj in program_ctx.name_map.keys(): if obj not in program_ctx.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr(candidate, 'im_class') not in program_ctx.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, program_ctx, {}, {}) return node, name, ns
def is_whitelisted(o, check_call_override=True, allow_namedtuple_subclass=False): """Checks whether an entity is whitelisted for use in graph mode. Examples of whitelisted entities include all members of the tensorflow package. Args: o: A Python entity. check_call_override: Reserved for internal use. When set to `False`, it disables the rule according to which classes are whitelisted if their __call__ method is whitelisted. allow_namedtuple_subclass: Reserved for internal use. When `True`, namedtuple subclasses are not whitelisted. Returns: Boolean """ # TODO(b/120224672): Fix this. if isinstance(o, functools.partial): # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since # functools.partial objects do not have a __module__ attribute. m = functools else: m = tf_inspect.getmodule(o) # Examples of callables that lack a __module__ property include builtins. if hasattr(m, '__name__'): for rule in config.CONVERSION_RULES: action = rule.get_action(m) if action == config.Action.CONVERT: logging.log(2, 'Not whitelisted: %s: %s', o, rule) return False elif action == config.Action.DO_NOT_CONVERT: logging.log(2, 'Whitelisted: %s: %s', o, rule) return True if tf_inspect.isgeneratorfunction(o): logging.warn( 'Entity %s appears to be a generator function. It will not be converted' ' by AutoGraph.', o) logging.log(2, 'Whitelisted: %s: generator functions are not converted', o) return True if (check_call_override and not tf_inspect.isclass(o) and hasattr(o, '__call__')): # Callable objects: whitelisted if their __call__ method is. # The type check avoids infinite recursion around the __call__ method # of function objects. if (type(o) != type(o.__call__)) and is_whitelisted(o.__call__): # pylint: disable=unidiomatic-typecheck logging.log(2, 'Whitelisted: %s: object __call__ whitelisted', o) return True owner_class = None if tf_inspect.ismethod(o): # Methods of whitelisted classes are also whitelisted, even if they are # bound via user subclasses. # # For example, suppose `tf.Foo` has a method called `bar`, and `baz` is # defined as below. `tf.Foo` is whitelisted. Then `baz.bar` is also # whitelisted. # # class Custom(tf.Foo): # pass # # baz = Custom() # # For the example above, if `Custom` did overload `bar`, then it would no # longer be whitelisted. owner_class = inspect_utils.getmethodclass(o) if owner_class is function.TfMethodTarget: owner_class = o.__self__.target_class if owner_class is not None: if issubclass(owner_class, unittest.TestCase): logging.log(2, 'Whitelisted: %s: method of TestCase subclass', o) return True owner_class = inspect_utils.getdefiningclass(o, owner_class) if is_whitelisted(owner_class, check_call_override=False, allow_namedtuple_subclass=True): logging.log(2, 'Whitelisted: %s: owner is whitelisted %s', o, owner_class) return True if inspect_utils.isnamedtuple(o): # Due to the way they're constructed, namedtuple types cannot be converted # because they don't expose source code. But we assume they are safe for # graph mode since they are just containers. if allow_namedtuple_subclass: if not any( inspect_utils.isnamedtuple(base) for base in o.__bases__): logging.log(2, 'Whitelisted: %s: named tuple', o) return True else: logging.log(2, 'Whitelisted: %s: named tuple or subclass', o) return True logging.log(2, 'Not whitelisted: %s: default rule', o) return False
def entity_to_graph(o, program_ctx, arg_values, arg_types): """Compile a Python entity into equivalent TensorFlow. The function will also recursively compile all the entities that `o` references, updating `dependency_cache`. This function is reentrant, and relies on dependency_cache to avoid generating duplicate code. Args: o: A Python entity. program_ctx: A ProgramContext object. arg_values: A dict containing value hints for symbols like function parameters. arg_types: A dict containing type hints for symbols like function parameters. Returns: A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. * namespace: A dict mapping all symbols visible to the converted entity, keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ logging.log(1, 'Converting %s', o) if tf_inspect.isclass(o): node, name, ns = class_to_graph(o, program_ctx) elif tf_inspect.isfunction(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) elif tf_inspect.ismethod(o): node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types) # TODO(mdan,yashkatariya): Remove when object conversion is implemented. elif hasattr(o, '__class__'): raise NotImplementedError( 'Object conversion is not yet supported. If you are ' 'trying to convert code that uses an existing object, ' 'try including the creation of that object in the ' 'conversion. For example, instead of converting the method ' 'of a class, try converting the entire class instead. ' 'See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/' 'contrib/autograph/README.md#using-the-functional-api ' 'for more information.') else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' 'supported for now.' % (o, type(o))) # TODO(mdan): This is temporary. it should be created using a converter. # TODO(mdan): The attribute should be added with a helper, not directly. # The helper can ensure there are no collisions. template = ''' entity.autograph_info__ = {} ''' node.extend(templates.replace(template, entity=name)) program_ctx.add_to_cache(o, node) if logging.has_verbosity(2): logging.log(2, 'Compiled output of %s:\n\n%s\n', o, compiler.ast_to_source(node)) if program_ctx.options.recursive: while True: candidate = None for obj in program_ctx.name_map.keys(): if obj not in program_ctx.dependency_cache: candidate = obj break if candidate is None: break if (hasattr(candidate, 'im_class') and getattr( candidate, 'im_class') not in program_ctx.partial_types): # Class members are converted with their objects, unless they're # only converted partially. continue entity_to_graph(candidate, program_ctx, {}, {}) return node, name, ns
def converted_call(f, owner, options, args, kwargs): """Compiles a function call inline. For internal use only.""" logging.log(1, 'Converted call: %s; owner: %s\n args: %s\n kwargs: %s\n', f, owner, args, kwargs) if owner is not None: if not isinstance(f, str): raise ValueError( 'When owner is specified, the function name must be specified as' ' a string: {}'.format(f)) # Special case when the owner is a 'super' object. In that case lookups of # dynamic attributes won't work. See # inspect_utils.SuperWrapperForDynamicAttrs. if isinstance(owner, super): owner = inspect_utils.SuperWrapperForDynamicAttrs(owner) f = getattr(owner, f) if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) # TODO(b/122265385): Remove this bypass. if ('wrapt' in sys.modules and hasattr(sys.modules['wrapt'], 'FunctionWrapper') and isinstance(f, sys.modules['wrapt'].FunctionWrapper)): logging.warn( 'Entity {} appears to be decorated by wrapt, which is not yet supported' ' by AutoGraph. The function will be called without transformation.' ' You may however apply AutoGraph before the decorator.'.format(f), 1) logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f) return f(*args, **kwargs) # Constructors are permanently whitelisted. # TODO(mdan): Toggle as experimental feature instead. # TODO(b/124016764): Remove this limitation. if tf_inspect.isclass(f): logging.log(2, 'Permanently whitelisted: %s: constructor', f) return f(*args, **kwargs) # Other built-in modules are permanently whitelisted. # TODO(mdan): Figure out how to do this consistently for all stdlib modules. if (f in collections.__dict__.values() or f in pdb.__dict__.values() or f in copy.__dict__.values()): logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f) return f(*args, **kwargs) # TODO(mdan): This needs cleanup. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): # TODO(mdan): This may be inconsistent in certain situations. # If the function had already been annotated with @tf.function, it # may be bound to the incorrect object. It's unclear if those situations # are possible, but if they happen, we need to check if f is bound # to a shim like WeakrefSelf and unpack it. # Args typically include `self`, as required by the conversion process. # When conversion is skipped, `self` is not necessary, because the # original bound method is being executed. This code removes it. if tf_inspect.ismethod(f) and args: f_self = inspect_utils.getmethodself(f) if args[0] is f_self: args = args[1:] return f(*args, **kwargs) # internal_convert_user_code is for example turned off when issuing a dynamic # call conversion from generated code while in nonrecursive mode. In that # case we evidently don't want to recurse, but we still have to convert # things like builtins. if not options.internal_convert_user_code: return f(*args, **kwargs) # TODO(mdan): Move this entire block inside to_graph. try: # Begin of transformation error guards # Unwrap functools.partial objects # TODO(mdan): Consider sharing unwrapping logic with tf_inspect. while isinstance(f, functools.partial): args = f.args + args new_kwargs = {} if f.keywords is not None: new_kwargs.update(f.keywords) new_kwargs.update(kwargs) kwargs = new_kwargs f = f.func if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f f_self = inspect_utils.getmethodself(f) # TODO(b/119246461): This may be more elegantly handled using __get__? if f_self is not None: # If this is a method call, it may or may not include self. # # Example when self is included: # converted_call(to_graph(foo.bar), foo) # # Example when self is not included: # super(...).foo(args) # if owner is not None and (not args or args[0] is not owner): effective_args = (owner,) + args else: # When the owner is not specified, use the result of # inspect_utils.getmethodclass. # TODO(b/119246461): Make sure an owner is always specified. if not args or args[0] is not f_self: effective_args = (f_self,) + args else: effective_args = (f_self,) + args[1:] partial_types = (f_self,) else: effective_args = args partial_types = () elif tf_inspect.isclass(f): # Constructors # Note: Until we support class constructurs, and enable whole-class # conversion with an experimental flag, this branch is dead code. # TODO(mdan): Consider removing unless there is a compelling use case. target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f,) + args partial_types = (f.__class__,) else: raise NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) arg_types = {} for name, arg in arg_values.items(): arg_class = arg.__class__ arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__,) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'],) logging.log(3, 'Partial types in conversion of %s: %s', target_entity, partial_types) converted_f = to_graph( target_entity, recursive=options.recursive, arg_values=arg_values, arg_types=arg_types, experimental_optional_features=options.optional_features, experimental_strip_decorators=options.strip_decorators, experimental_verbose=options.verbose, experimental_partial_types=partial_types) if logging.has_verbosity(2): logging.log(2, 'Defaults of %s : %s', converted_f, converted_f.__defaults__) callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs) formatted_callargs = '\n'.join( ' {}: {}'.format(k, v) for k, v in callargs.items()) logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs) # TODO(mdan): Reduce this list. except (errors.AutoGraphError, AssertionError, AttributeError, IndexError, KeyError, NameError, NotImplementedError, SyntaxError, TypeError, ValueError, IOError) as e: logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True) logging.warn( 'Entity %s could not be transformed and will be staged without change.' ' Error details can be found in the logs when running with the env' ' variable AUTOGRAPH_VERBOSITY=5. Please report this to the AutoGraph' ' team. Cause: %s', target_entity, e) return f(*args, **kwargs) result = converted_f(*effective_args, **kwargs) # The converted function's closure is simply inserted into the function's # module __dict__. Since modules are permanently cached, that results in # leaking the entire closure. # Normally, it's not safe to delete the module because that may release said # closure as well. However, in the case of converted_call we are certain the # function will not be executed again, so the closure should no longer be # needed so long as the function doesn't return any executable code. # TODO(mdan): Attach the closure properly, using cells. if all(map(_is_not_callable, nest.flatten(result))): del sys.modules[converted_f.__module__] return result
def converted_call(f, options, *args, **kwargs): """Compiles a function call inline. For internal use only.""" # TODO(mdan): This needs cleanup. # In particular, we may want to avoid renaming functions altogether. if not options.force_conversion and conversion.is_whitelisted_for_graph(f): return f(*args, **kwargs) unknown_arg_value = object() # Sentinel for arguments of unknown value if inspect_utils.isbuiltin(f): return py_builtins.overload_of(f)(*args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f arg_map_target = f effective_args = args f_class = inspect_utils.getmethodclass(f) if f_class is not None: partial_types = (f_class, ) else: partial_types = () elif tf_inspect.isclass(f): # Constructors target_entity = f arg_map_target = f.__init__ effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): # Callable objects target_entity = f.__call__ arg_map_target = f.__call__ effective_args = (f, ) + args partial_types = (f.__class__, ) else: NotImplementedError('unknown callable type "%s"' % type(f)) arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs) for name, arg in arg_values.items(): if arg is unknown_arg_value: continue arg_class = arg.__class__ # If arg_value_hints specifies any name, use that instead. if name not in options.arg_types: options.arg_types[name] = (arg_class.__name__, arg_class) # When called from within a decorator, this is the only indication that # the function is a method - it appears that the decorator is applied # before the method is bound. if not partial_types: if 'self' in arg_values: if tf_inspect.isclass(arg_values['self'].__class__): partial_types = (arg_values['self'].__class__, ) elif 'cls' in arg_values: if tf_inspect.isclass(arg_values['cls']): partial_types = (arg_values['cls'], ) converted_f = to_graph(target_entity, recursive=options.recursive, verbose=options.verbose, arg_values=arg_values, arg_types=options.arg_types, partial_types=partial_types, strip_decorators=options.strip_decorators) return converted_f(*effective_args, **kwargs)
def populate_deserializable_objects(): """Populates dict ALL_OBJECTS with every built-in initializer. """ global LOCAL if not hasattr(LOCAL, 'ALL_OBJECTS'): LOCAL.ALL_OBJECTS = {} LOCAL.GENERATED_WITH_V2 = None if LOCAL.ALL_OBJECTS and LOCAL.GENERATED_WITH_V2 == tf2.enabled(): # Objects dict is already generated for the proper TF version: # do nothing. return LOCAL.ALL_OBJECTS = {} LOCAL.GENERATED_WITH_V2 = tf2.enabled() # Compatibility aliases (need to exist in both V1 and V2). LOCAL.ALL_OBJECTS['ConstantV2'] = initializers_v2.Constant LOCAL.ALL_OBJECTS['GlorotNormalV2'] = initializers_v2.GlorotNormal LOCAL.ALL_OBJECTS['GlorotUniformV2'] = initializers_v2.GlorotUniform LOCAL.ALL_OBJECTS['HeNormalV2'] = initializers_v2.HeNormal LOCAL.ALL_OBJECTS['HeUniformV2'] = initializers_v2.HeUniform LOCAL.ALL_OBJECTS['IdentityV2'] = initializers_v2.Identity LOCAL.ALL_OBJECTS['LecunNormalV2'] = initializers_v2.LecunNormal LOCAL.ALL_OBJECTS['LecunUniformV2'] = initializers_v2.LecunUniform LOCAL.ALL_OBJECTS['OnesV2'] = initializers_v2.Ones LOCAL.ALL_OBJECTS['OrthogonalV2'] = initializers_v2.Orthogonal LOCAL.ALL_OBJECTS['RandomNormalV2'] = initializers_v2.RandomNormal LOCAL.ALL_OBJECTS['RandomUniformV2'] = initializers_v2.RandomUniform LOCAL.ALL_OBJECTS['TruncatedNormalV2'] = initializers_v2.TruncatedNormal LOCAL.ALL_OBJECTS['VarianceScalingV2'] = initializers_v2.VarianceScaling LOCAL.ALL_OBJECTS['ZerosV2'] = initializers_v2.Zeros # Out of an abundance of caution we also include these aliases that have # a non-zero probability of having been included in saved configs in the past. LOCAL.ALL_OBJECTS['glorot_normalV2'] = initializers_v2.GlorotNormal LOCAL.ALL_OBJECTS['glorot_uniformV2'] = initializers_v2.GlorotUniform LOCAL.ALL_OBJECTS['he_normalV2'] = initializers_v2.HeNormal LOCAL.ALL_OBJECTS['he_uniformV2'] = initializers_v2.HeUniform LOCAL.ALL_OBJECTS['lecun_normalV2'] = initializers_v2.LecunNormal LOCAL.ALL_OBJECTS['lecun_uniformV2'] = initializers_v2.LecunUniform if tf2.enabled(): # For V2, entries are generated automatically based on the content of # initializers_v2.py. v2_objs = {} base_cls = initializers_v2.Initializer generic_utils.populate_dict_with_module_objects( v2_objs, [initializers_v2], obj_filter=lambda x: inspect.isclass(x) and issubclass(x, base_cls)) for key, value in v2_objs.items(): LOCAL.ALL_OBJECTS[key] = value # Functional aliases. LOCAL.ALL_OBJECTS[generic_utils.to_snake_case(key)] = value else: # V1 initializers. v1_objs = { 'Constant': init_ops.Constant, 'GlorotNormal': init_ops.GlorotNormal, 'GlorotUniform': init_ops.GlorotUniform, 'Identity': init_ops.Identity, 'Ones': init_ops.Ones, 'Orthogonal': init_ops.Orthogonal, 'VarianceScaling': init_ops.VarianceScaling, 'Zeros': init_ops.Zeros, 'HeNormal': initializers_v1.HeNormal, 'HeUniform': initializers_v1.HeUniform, 'LecunNormal': initializers_v1.LecunNormal, 'LecunUniform': initializers_v1.LecunUniform, 'RandomNormal': initializers_v1.RandomNormal, 'RandomUniform': initializers_v1.RandomUniform, 'TruncatedNormal': initializers_v1.TruncatedNormal, } for key, value in v1_objs.items(): LOCAL.ALL_OBJECTS[key] = value # Functional aliases. LOCAL.ALL_OBJECTS[generic_utils.to_snake_case(key)] = value # More compatibility aliases. LOCAL.ALL_OBJECTS['normal'] = LOCAL.ALL_OBJECTS['random_normal'] LOCAL.ALL_OBJECTS['uniform'] = LOCAL.ALL_OBJECTS['random_uniform'] LOCAL.ALL_OBJECTS['one'] = LOCAL.ALL_OBJECTS['ones'] LOCAL.ALL_OBJECTS['zero'] = LOCAL.ALL_OBJECTS['zeros']