def replace(template, **replacements): """Replace placeholders in a Python template. AST Name and Tuple nodes always receive the context that inferred from the template. However, when replacing more complex nodes (that can potentially contain Name children), then the caller is responsible for setting the appropriate context. Args: template: A string representing Python code. Any symbol name can be used that appears in the template code can be used as placeholder. **replacements: A mapping from placeholder names to (lists of) AST nodes that these placeholders will be replaced by. String values are also supported as a shorthand for AST Name nodes with the respective ID. Returns: An AST node or list of AST nodes with the replacements made. If the template was a function, a list will be returned. If the template was a node, the same node will be returned. If the template was a string, an AST node will be returned (a `Module` node in the case of a multi-line string, an `Expr` node otherwise). Raises: ValueError: if the arguments are incorrect. """ if not isinstance(template, str): raise ValueError('Expected string template, got %s' % type(template)) tree = parser.parse_str(textwrap.dedent(template)) for k in replacements: replacements[k] = _convert_to_ast(replacements[k]) results = ReplaceTransformer(replacements).visit(tree).body if isinstance(results, list): return [qual_names.resolve(r) for r in results] return qual_names.resolve(results)
def test_subscript_resolve(self): samples = """ x[i] x[i.b] a.b[c] a.b[x.y] a[z[c]] a[b[c[d]]] a[b].c a.b.c[d].e.f a.b[c[d]].e.f a.b[c[d.e.f].g].h """ nodes = resolve(parser.parse_str(textwrap.dedent(samples))) nodes = tuple(n.value for n in nodes.body) self.assertQNStringIs(nodes[0], 'x[i]') self.assertQNStringIs(nodes[1], 'x[i.b]') self.assertQNStringIs(nodes[2], 'a.b[c]') self.assertQNStringIs(nodes[3], 'a.b[x.y]') self.assertQNStringIs(nodes[4], 'a[z[c]]') self.assertQNStringIs(nodes[5], 'a[b[c[d]]]') self.assertQNStringIs(nodes[6], 'a[b].c') self.assertQNStringIs(nodes[7], 'a.b.c[d].e.f') self.assertQNStringIs(nodes[8], 'a.b[c[d]].e.f') self.assertQNStringIs(nodes[9], 'a.b[c[d.e.f].g].h')
def parse_and_analyze(self, test_fn, namespace, namer=None, arg_types=None, include_type_analysis=True, owner_type=None, recursive=True, autograph_decorators=()): node, source = parser.parse_entity(test_fn) if namer is None: namer = FakeNamer() program_ctx = converter.ProgramContext( recursive=recursive, autograph_decorators=autograph_decorators, partial_types=None, autograph_module=None, uncompiled_modules=config.DEFAULT_UNCOMPILED_MODULES) entity_info = transformer.EntityInfo(source_code=source, source_file='<fragment>', namespace=namespace, arg_values=None, arg_types=arg_types, owner_type=owner_type) ctx = converter.EntityContext(namer, entity_info, program_ctx) node = qual_names.resolve(node) node = activity.resolve(node, entity_info) node = live_values.resolve(node, entity_info, {}) if include_type_analysis: node = type_info.resolve(node, entity_info) node = live_values.resolve(node, entity_info, {}) self.ctx = ctx return node
def standard_analysis(node, context, is_initial=False): """Performs a complete static analysis of the given code. Args: node: ast.AST context: converter.EntityContext is_initial: bool, whether this is the initial analysis done on the input source code Returns: ast.AST, same as node, with the static analysis annotations added """ # TODO(mdan): Clear static analysis here. # TODO(mdan): Consider not running all analyses every time. # TODO(mdan): Don't return a node because it's modified by reference. graphs = cfg.build(node) node = qual_names.resolve(node) node = activity.resolve(node, context.info, None) node = reaching_definitions.resolve(node, context.info, graphs, AnnotatedDef) node = liveness.resolve(node, context.info, graphs) node = live_values.resolve(node, context.info, config.PYTHON_LITERALS) node = type_info.resolve(node, context.info) # This second call allows resolving first-order class attributes. node = live_values.resolve(node, context.info, config.PYTHON_LITERALS) if is_initial: anno.dup( node, { anno.Static.DEFINITIONS: anno.Static.ORIG_DEFINITIONS, }, ) return node
def parse_and_analyze(self, test_fn, namespace, namer=None, arg_types=None, include_type_analysis=True, owner_type=None, recursive=True): node, source = parser.parse_entity(test_fn) ctx = context.EntityContext( namer=namer or FakeNamer(), source_code=source, source_file=None, namespace=namespace, arg_values=None, arg_types=arg_types, owner_type=owner_type, recursive=recursive, type_annotation_func=utils.set_element_type) node = qual_names.resolve(node) node = activity.resolve(node, ctx) node = live_values.resolve(node, ctx, {}) if include_type_analysis: node = type_info.resolve(node, ctx) node = live_values.resolve(node, ctx, {}) self.ctx = ctx return node
def _apply_transformer(node, context, converter_module): # TODO(mdan): Clear static analysis here. node = qual_names.resolve(node) node = activity.resolve(node, context.info, None) node = live_values.resolve(node, context.info, config.PYTHON_LITERALS) node = type_info.resolve(node, context.info) node = converter_module.transform(node, context) return node
def test_rename_symbols_basic(self): node = parser.parse_str('a + b') node = qual_names.resolve(node) node = ast_util.rename_symbols( node, {qual_names.QN('a'): qual_names.QN('renamed_a')}) self.assertIsInstance(node.body[0].value.left.id, str) self.assertEqual(compiler.ast_to_source(node).strip(), 'renamed_a + b')
def test_rename_symbols_attributes(self): node = parser.parse_str('b.c = b.c.d') node = qual_names.resolve(node) node = ast_util.rename_symbols( node, {qual_names.from_str('b.c'): qual_names.QN('renamed_b_c')}) source, _ = compiler.ast_to_source(node) self.assertEqual(source.strip(), 'renamed_b_c = renamed_b_c.d')
def test_rename_symbols_attributes(self): node = parser.parse_str('b.c = b.c.d') node = qual_names.resolve(node) node = ast_util.rename_symbols( node, {qual_names.from_str('b.c'): qual_names.QN('renamed_b_c')}) source = compiler.ast_to_source(node) self.assertEqual(source.strip(), 'renamed_b_c = renamed_b_c.d')
def test_rename_symbols_basic(self): node = parser.parse_str('a + b') node = qual_names.resolve(node) node = ast_util.rename_symbols( node, {qual_names.QN('a'): qual_names.QN('renamed_a')}) self.assertIsInstance(node.body[0].value.left.id, str) source = compiler.ast_to_source(node) self.assertEqual(source.strip(), 'renamed_a + b')
def test_rename_symbols_annotations(self): node = parser.parse_str('a[i]') node = qual_names.resolve(node) anno.setanno(node, 'foo', 'bar') orig_anno = anno.getanno(node, 'foo') node = ast_util.rename_symbols( node, {qual_names.QN('a'): qual_names.QN('b')}) self.assertIs(anno.getanno(node, 'foo'), orig_anno)
def test_rename_symbols_annotations(self): node = parser.parse_str('a[i]') node = qual_names.resolve(node) anno.setanno(node, 'foo', 'bar') orig_anno = anno.getanno(node, 'foo') node = ast_util.rename_symbols(node, {qual_names.QN('a'): qual_names.QN('b')}) self.assertIs(anno.getanno(node, 'foo'), orig_anno)
def _parse_and_analyze(self, test_fn): node, source = parser.parse_entity(test_fn) entity_info = transformer.EntityInfo( source_code=source, source_file=None, namespace={}, arg_values=None, arg_types=None, owner_type=None) node = qual_names.resolve(node) return node, entity_info
def _parse_and_analyze(self, test_fn): node, source = parser.parse_entity(test_fn) entity_info = transformer.EntityInfo(source_code=source, source_file=None, namespace={}, arg_values=None, arg_types=None, owner_type=None) node = qual_names.resolve(node) node = activity.resolve(node, entity_info) return node, entity_info
def _parse_and_analyze(self, test_fn, namespace, arg_types=None): arg_types = arg_types or {} node, source = parser.parse_entity(test_fn) ctx = context.EntityContext(namer=None, source_code=source, source_file=None, namespace=namespace, arg_values=None, arg_types=arg_types, owner_type=None, recursive=True) node = qual_names.resolve(node) return node, ctx
def _parse_and_analyze(self, test_fn, namespace, arg_types=None): arg_types = arg_types or {} node, source = parser.parse_entity(test_fn) ctx = context.EntityContext( namer=None, source_code=source, source_file=None, namespace=namespace, arg_values=None, arg_types=arg_types, owner_type=None, recursive=True) node = qual_names.resolve(node) return node, ctx
def _parse_and_analyze(self, test_fn): node, source = parser.parse_entity(test_fn) entity_info = transformer.EntityInfo( source_code=source, source_file=None, namespace={}, arg_values=None, arg_types=None, owner_type=None) node = qual_names.resolve(node) node = activity.resolve(node, entity_info) graphs = cfg.build(node) node = reaching_definitions.resolve(node, entity_info, graphs, reaching_definitions.Definition) return node
def replace_as_expression(template, **replacements): """Variant of replace that generates expressions, instead of code blocks.""" replacement = replace(template, **replacements) if len(replacement) != 1: raise ValueError( 'single expression expected; for more general templates use replace') node = replacement[0] node = qual_names.resolve(node) if isinstance(node, gast.Expr): return node.value elif isinstance(node, gast.Name): return node raise ValueError( 'the template is expected to generate an expression or a name node;' ' instead found %s' % node)
def test_function_calls(self): samples = """ a.b a.b() a().b z[i] z[i]() z()[i] """ nodes = resolve(parser.parse_str(textwrap.dedent(samples))) nodes = tuple(n.value for n in nodes.body) self.assertQNStringIs(nodes[0], 'a.b') self.assertQNStringIs(nodes[1].func, 'a.b') self.assertQNStringIs(nodes[2].value.func, 'a') self.assertQNStringIs(nodes[3], 'z[i]') self.assertQNStringIs(nodes[4].func, 'z[i]') self.assertQNStringIs(nodes[5].value.func, 'z')
def _parse_and_analyze(self, test_fn, namespace, arg_types=None): node, source = parser.parse_entity(test_fn) entity_info = transformer.EntityInfo( source_code=source, source_file=None, namespace=namespace, arg_values=None, arg_types=arg_types, owner_type=None) node = qual_names.resolve(node) node = activity.resolve(node, entity_info) node = live_values.resolve(node, entity_info, {}) node = type_info.resolve(node, entity_info) node = live_values.resolve(node, entity_info, {}) return node
def _parse_and_analyze(self, test_fn, namespace, literals=None, arg_types=None): literals = literals or {} node, source = parser.parse_entity(test_fn) entity_info = transformer.EntityInfo( source_code=source, source_file=None, namespace=namespace, arg_values=None, arg_types=arg_types, owner_type=None) node = qual_names.resolve(node) node = activity.resolve(node, entity_info) node = live_values.resolve(node, entity_info, literals) node = type_info.resolve(node, entity_info) node = live_values.resolve(node, entity_info, literals) return node
def test_resolve(self): samples = """ a a.b (c, d.e) [f, (g.h.i)] j(k, l) """ nodes = resolve(parser.parse_str(textwrap.dedent(samples))) nodes = tuple(n.value for n in nodes.body) self.assertQNStringIs(nodes[0], 'a') self.assertQNStringIs(nodes[1], 'a.b') self.assertQNStringIs(nodes[2].elts[0], 'c') self.assertQNStringIs(nodes[2].elts[1], 'd.e') self.assertQNStringIs(nodes[3].elts[0], 'f') self.assertQNStringIs(nodes[3].elts[1], 'g.h.i') self.assertQNStringIs(nodes[4].func, 'j') self.assertQNStringIs(nodes[4].args[0], 'k') self.assertQNStringIs(nodes[4].args[1], 'l')
def parse_and_analyze(self, test_fn, namespace, namer=None, arg_types=None, include_type_analysis=True, owner_type=None, recursive=True, autograph_decorators=()): node, source = parser.parse_entity(test_fn) if namer is None: namer = FakeNamer() program_ctx = converter.ProgramContext( recursive=recursive, autograph_decorators=autograph_decorators, partial_types=None, autograph_module=None, uncompiled_modules=config.DEFAULT_UNCOMPILED_MODULES) entity_info = transformer.EntityInfo( source_code=source, source_file='<fragment>', namespace=namespace, arg_values=None, arg_types=arg_types, owner_type=owner_type) ctx = converter.EntityContext(namer, entity_info, program_ctx) node = qual_names.resolve(node) node = activity.resolve(node, entity_info) node = live_values.resolve(node, entity_info, {}) if include_type_analysis: node = type_info.resolve(node, entity_info) node = live_values.resolve(node, entity_info, {}) self.ctx = ctx return node
def test_rename_symbols(self): node = ast.Tuple([ ast.Name('a', ast.Load()), ast.Name('b', ast.Load()), ast.Attribute(ast.Name('b', None), 'c', ast.Store()), ast.Attribute( ast.Attribute(ast.Name('b', None), 'c', ast.Load()), 'd', None) ], None) node = qual_names.resolve(node) node = ast_util.rename_symbols( node, { qual_names.QN('a'): qual_names.QN('renamed_a'), qual_names.QN(qual_names.QN('b'), attr='c'): qual_names.QN('renamed_b_c'), }) self.assertEqual(node.elts[0].id, 'renamed_a') self.assertTrue(isinstance(node.elts[0].ctx, ast.Load)) self.assertEqual(node.elts[1].id, 'b') self.assertEqual(node.elts[2].id, 'renamed_b_c') self.assertTrue(isinstance(node.elts[2].ctx, ast.Store)) self.assertEqual(node.elts[3].value.id, 'renamed_b_c') self.assertTrue(isinstance(node.elts[3].value.ctx, ast.Load))
def test_rename_symbols(self): node = ast.Tuple([ ast.Name('a', ast.Load()), ast.Name('b', ast.Load()), ast.Attribute(ast.Name('b', None), 'c', ast.Store()), ast.Attribute(ast.Attribute(ast.Name('b', None), 'c', ast.Load()), 'd', None) ], None) node = qual_names.resolve(node) node = ast_util.rename_symbols( node, { qual_names.QN('a'): qual_names.QN('renamed_a'), qual_names.QN(qual_names.QN('b'), attr='c'): qual_names.QN('renamed_b_c'), }) self.assertEqual(node.elts[0].id, 'renamed_a') self.assertTrue(isinstance(node.elts[0].ctx, ast.Load)) self.assertEqual(node.elts[1].id, 'b') self.assertEqual(node.elts[2].id, 'renamed_b_c') self.assertTrue(isinstance(node.elts[2].ctx, ast.Store)) self.assertEqual(node.elts[3].value.id, 'renamed_b_c') self.assertTrue(isinstance(node.elts[3].value.ctx, ast.Load))
def _static_analysis_pass(node, ctx): node = qual_names.resolve(node) node = activity.resolve(node, ctx, None) node = live_values.resolve(node, ctx, config.PYTHON_LITERALS) node = type_info.resolve(node, ctx) return node
def class_to_graph(c, program_ctx): """Specialization of `entity_to_graph` for classes.""" converted_members = {} method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m ) members = tf_inspect.getmembers(c, predicate=method_filter) if not members: raise ValueError('Cannot convert %s: it has no member methods.' % c) class_namespace = {} for _, m in members: # Only convert the members that are directly defined by the class. if inspect_utils.getdefiningclass(m, c) is not c: continue node, _, namespace = function_to_graph( m, program_ctx=program_ctx, arg_values={}, arg_types={'self': (c.__name__, c)}, owner_type=c) if class_namespace is None: class_namespace = namespace else: class_namespace.update(namespace) converted_members[m] = node namer = program_ctx.new_namer(class_namespace) class_name = namer.compiled_class_name(c.__name__, c) # TODO(mdan): This needs to be explained more thoroughly. # Process any base classes: if the sueprclass if of a whitelisted type, an # absolute import line is generated. Otherwise, it is marked for conversion # (as a side effect of the call to namer.compiled_class_name() followed by # program_ctx.update_name_map(namer)). output_nodes = [] renames = {} bases = [] for base in c.__bases__: if isinstance(object, base): bases.append('object') continue if is_whitelisted_for_graph(base): alias = namer.new_symbol(base.__name__, ()) output_nodes.append( gast.ImportFrom( module=base.__module__, names=[gast.alias(name=base.__name__, asname=alias)], level=0)) else: # This will trigger a conversion into a class with this name. alias = namer.compiled_class_name(base.__name__, base) bases.append(alias) renames[qual_names.QN(base.__name__)] = qual_names.QN(alias) program_ctx.update_name_map(namer) # Generate the definition of the converted class. output_nodes.append( gast.ClassDef(class_name, bases=bases, keywords=[], body=list(converted_members.values()), decorator_list=[])) node = gast.Module(output_nodes) # Make a final pass to replace references to the class or its base classes. # Most commonly, this occurs when making super().__init__() calls. # TODO(mdan): Making direct references to superclass' superclass will fail. node = qual_names.resolve(node) renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name) node = ast_util.rename_symbols(node, renames) return node, class_name, class_namespace
def class_to_graph(c, program_ctx): """Specialization of `entity_to_graph` for classes.""" converted_members = {} method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m) members = tf_inspect.getmembers(c, predicate=method_filter) if not members: raise ValueError('Cannot convert %s: it has no member methods.' % c) class_namespace = {} for _, m in members: # Only convert the members that are directly defined by the class. if inspect_utils.getdefiningclass(m, c) is not c: continue node, _, namespace = function_to_graph( m, program_ctx=program_ctx, arg_values={}, arg_types={'self': (c.__name__, c)}, owner_type=c, rewrite_errors=False) if class_namespace is None: class_namespace = namespace else: class_namespace.update(namespace) converted_members[m] = node[0] namer = program_ctx.new_namer(class_namespace) class_name = namer.compiled_class_name(c.__name__, c) # TODO(mdan): This needs to be explained more thoroughly. # Process any base classes: if the sueprclass if of a whitelisted type, an # absolute import line is generated. Otherwise, it is marked for conversion # (as a side effect of the call to namer.compiled_class_name() followed by # program_ctx.update_name_map(namer)). output_nodes = [] renames = {} base_names = [] for base in c.__bases__: if isinstance(object, base): base_names.append('object') continue if is_whitelisted_for_graph(base): alias = namer.new_symbol(base.__name__, ()) output_nodes.append( gast.ImportFrom( module=base.__module__, names=[gast.alias(name=base.__name__, asname=alias)], level=0)) else: # This will trigger a conversion into a class with this name. alias = namer.compiled_class_name(base.__name__, base) base_names.append(alias) renames[qual_names.QN(base.__name__)] = qual_names.QN(alias) program_ctx.update_name_map(namer) # Generate the definition of the converted class. bases = [gast.Name(n, gast.Load(), None) for n in base_names] class_def = gast.ClassDef( class_name, bases=bases, keywords=[], body=list(converted_members.values()), decorator_list=[]) # Make a final pass to replace references to the class or its base classes. # Most commonly, this occurs when making super().__init__() calls. # TODO(mdan): Making direct references to superclass' superclass will fail. class_def = qual_names.resolve(class_def) renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name) class_def = ast_util.rename_symbols(class_def, renames) output_nodes.append(class_def) return output_nodes, class_name, class_namespace