def parse(code): """String -> AST Parse the string and return its AST representation. May raise a ParseError exception. """ added_newline = False if not code.endswith("\n"): code += "\n" added_newline = True try: drv = driver.Driver(pygram.python_grammar, pytree.convert) result = drv.parse_string(code, True) except ParseError: log.debug("Had problems parsing:\n%s\n" % quoted_block(code)) raise # Always return a Node, not a Leaf. if isinstance(result, Leaf): result = Node(syms.file_input, [result]) result.added_newline = added_newline return result
def fix_not_a_in_b(node: LN, capture: Capture, filename: Filename): capture["element"].parent = None capture["collection"].parent = None new_comparison = Node( comparison, [ capture["element"], Node(comp_op, [Leaf(1, "not", prefix=" "), Leaf(1, "in", prefix=" ")]), capture["collection"], ], ) new_comparison.parent = node.parent return new_comparison
def get_annotated_param(node, arg, *, missing_ok=False): if node.type not in (token.NAME, syms.tname): raise NotImplementedError(f"unexpected node token: `{node}`") actual_ann = None if node.type == syms.tname: actual_ann = node.children[2] node = node.children[0] if not isinstance(node, Leaf) or arg.arg != node.value: raise ValueError( f".pyi file expects argument {arg.arg!r} next but argument " + f"{minimize_whitespace(str(node))!r} found in source") if arg.annotation is None: if actual_ann is None: if missing_ok: return new(node) raise ValueError( f".pyi file is missing annotation for {arg.arg!r} and source " + f"doesn't provide it either") ann = new(actual_ann) else: ann = convert_annotation(arg.annotation) ann.prefix = ' ' if actual_ann is not None and actual_ann != ann: ann_str = minimize_whitespace(str(ann)) actual_ann_str = minimize_whitespace(str(actual_ann)) raise ValueError( f"incompatible annotation for {arg.arg!r}. Expected: " + f"{ann_str!r}, actual: {actual_ann_str!r}") return Node(syms.tname, [new(node), new(_colon), ann])
def add_future(node, symbol): root = fixer_util.find_root(node) for idx, node in enumerate(root.children): if node.type == syms.simple_stmt and \ len(node.children) > 0 and node.children[0].type == token.STRING: # skip over docstring continue names = check_future_import(node) if not names: # not a future statement; need to insert before this break if symbol in names: # already imported return import_ = fixer_util.FromImport('__future__', [Leaf(token.NAME, symbol, prefix=" ")]) # Place after any comments or whitespace. (copyright, shebang etc.) import_.prefix = node.prefix node.prefix = '' children = [import_, fixer_util.Newline()] root.insert_child(idx, Node(syms.simple_stmt, children))
def append_credentials(src, access_key, secret_key, target='CREDENTIALS', comment='Added by cornac'): if isinstance(src, str): if not src.endswith('\n'): src += '\n' src = parse_py(src) for varname, atom in iter_assign(src): if varname != target: continue if len(atom.children) == 2: # Atom is an empty dict. dictmaker = Node(syms.dictsetmaker, []) atom.children.insert(1, dictmaker) if atom.children[1].type != syms.dictsetmaker: raise ValueError(f"{target} is not a plain dict.") append_to_dict(atom, key=access_key, value=secret_key, comment=comment) break else: raise ValueError(f"{target} is not set.") return src
def _c_tuple(tup): contents = [_convert_annotation(elt) for elt in tup.elts] for index in range(len(contents) - 1, 0, -1): contents[index].prefix = " " contents.insert(index, new(_comma)) return Node(syms.subscriptlist, contents)
def future_import2(feature, node): """ An alternative to future_import() which might not work ... """ root = find_root(node) if does_tree_import(u"__future__", feature, node): return insert_pos = 0 for idx, node in enumerate(root.children): if node.type == syms.simple_stmt and node.children and \ node.children[0].type == token.STRING: insert_pos = idx + 1 break for thing_after in root.children[insert_pos:]: if thing_after.type == token.NEWLINE: insert_pos += 1 continue prefix = thing_after.prefix thing_after.prefix = u"" break else: prefix = u"" import_ = FromImport(u"__future__", [Leaf(token.NAME, feature, prefix=u" ")]) children = [import_, Newline()] root.insert_child(insert_pos, Node(syms.simple_stmt, children, prefix=prefix))
def add_global_assignment_after_imports(_name, assignment, node): """ Big copy paste + modification from touch_import """ root = find_root(node) if find_binding(_name, root): return # figure out where to insert the assignment. # First try to find the first import and then skip to the last one. insert_pos = offset = 0 for idx, node in enumerate(root.children): if not is_import_ish_stmt(node): continue for offset, node2 in enumerate(root.children[idx:]): if not is_import_ish_stmt(node2): break insert_pos = idx + offset break # if there are no imports where we can insert, find the docstring. # if that also fails, we stick to the beginning of the file if insert_pos == 0: for idx, node in enumerate(root.children): if (node.type == syms.simple_stmt and node.children and node.children[0].type == token.STRING): insert_pos = idx + 1 break children = [assignment, Newline()] root.insert_child(insert_pos, Node(syms.simple_stmt, children))
def transform(self, node, results): args = [] item_id = 0 while item_id < len(node.children): if (node.children[item_id].__class__.__name__ == "Leaf"): if node.children[item_id].value == ':': #加花括号 args += [ Leaf(token.LBRACE, '{'), node.children[item_id + 1].clone(), Leaf(token.RBRACE, '}'), Leaf(4, '\r\n') ] item_id += 1 #if,elif处理 elif (node.children[item_id].value == 'elif') | (node.children[item_id].value == 'if'): if node.children[item_id].value == 'elif': leaf_temp = Leaf(1, 'if') leaf_temp.prefix = " " args += [Leaf(1, 'else'), leaf_temp] else: args += [node.children[item_id].clone()] args += [Leaf(7, '(')] args += [node.children[item_id + 1].clone()] args += [Leaf(8, ')')] item_id += 1 else: args += [node.children[item_id].clone()] else: print("\nFixerError_if\n") item_id += 1 new = Node(node.type, args) new.prefix = node.prefix return new
def __insertImport(self, module, offset): if self.__coreImport is None: return importLine = Node(syms.simple_stmt, [ Node(syms.import_name, [ Leaf(token.NAME, u"import"), Leaf(token.NAME, module, prefix=" ") ]), Newline() ]) self.__coreImport.parent.insert_child( self.__coreImport.parent.children.index(self.__coreImport) + offset, importLine)
def transform(self, node, results): u""" Call __builtins__.long() with the value and the base of the value. This works because 0b10 is int("10", 2), 0o10 is int("10", 8), etc. """ val = node.value base_ = base(val) if base_ == 8: assert val.strip().startswith(u"0o") or \ val.strip().startswith( u"0O"), u"Invalid format for octal literal" node.changed() node.value = u"".join((u"0", val[2:])) elif base_ == 2: assert val.startswith(u"0") and val[1] in u"bB", \ u"Invalid format for binary literal" # __builtins__.long func_name = Node(syms.power, Attr(Name(u"__builtins__"), Name(u"long"))) # ("...", 2) func_args = [ String(u"".join((u"\"", val.strip()[2:], u"\""))), Comma(), Number(2, prefix=u" ") ] new_node = Call(func_name, func_args, node.prefix) return new_node
def suitify(parent): u""" Turn the stuff after the first colon in parent's children into a suite, if it wasn't already """ for node in parent.children: if node.type == syms.suite: # already in the prefered format, do nothing return # One-liners have no suite node, we have to fake one up for i, node in enumerate(parent.children): if node.type == token.COLON: break else: raise ValueError(u"No class suite and no ':'!") # Move everything into a suite node suite = Node(syms.suite, [ Newline(), Leaf(token.INDENT, indentation(node) + indentation_step(node)) ]) one_node = parent.children[i + 1] one_node.remove() one_node.prefix = u'' suite.append_child(one_node) parent.append_child(suite)
def transform(self, node, results): unifunc = results["unifunc"] strfunc = Name("__str__", prefix=unifunc.prefix) unifunc.replace(strfunc) klass = node.clone() klass.prefix = '\n' + find_indentation(node) decorator = Node( syms.decorator, [Leaf(token.AT, "@"), Name('python_2_unicode_compatible')]) decorated = Node(syms.decorated, [decorator, klass], prefix=node.prefix) node.replace(decorated) touch_import('django.utils.encoding', 'python_2_unicode_compatible', decorated)
def ImportAsName(name, as_name, prefix=None): new_name = Name(name) new_as = Name(u"as", prefix=u" ") new_as_name = Name(as_name, prefix=u" ") new_node = Node(syms.import_as_name, [new_name, new_as, new_as_name]) if prefix is not None: new_node.prefix = prefix return new_node
def get_dotted_import_replacement(self, name_node, attr_node, mapping=MAPPING, renamed=None): u""" For (http, client) given and httplib being the correct replacement, returns (httplib as client, None) For (test, support) given and test.test_support being the replacement, returns (test, test_support as support) """ full_name = name_node.value + u'.' + attr_node.value replacement = mapping[full_name] if u'.' in replacement: new_name, new_attr = replacement.split(u'.') if renamed is None: return Name(new_name, prefix=name_node.prefix), Node(syms.dotted_as_name, [Name(new_attr, prefix=attr_node.prefix), Name(u'as', prefix=u" "), attr_node.clone()]) else: return Name(new_name, prefix=name_node.prefix), Name(new_attr, prefix=attr_node.prefix) else: return Node(syms.dotted_as_name, [Name(replacement, prefix=name_node.prefix), Name(u'as', prefix=u' '), Name(attr_node.value, prefix=attr_node.prefix)]), None
def Def(name, args, *body, prefix=""): return Node(syms.funcdef, [ Name("def", prefix=prefix), maybe_name(name), Node(syms.parameters, [ LParen(), args, RParen(), ]), Colon(), Node(syms.suite, [ Newline(), Indent(), *body, Dedent(), ]) ])
def _Parse(cls, code): """ Parses the given code string returning its lib2to3 AST tree. :return lib2to3.AST: Returns the lib2to3 AST. """ def _GetLastLeaf(node): from lib2to3.pytree import Leaf r_leaf = node while not isinstance(r_leaf, Leaf): r_leaf = r_leaf.children[-1] return r_leaf # Prioritary import. # Other imports from zerotk.reraiseit import reraise from lib2to3 import pygram, pytree from lib2to3.pgen2 import driver from lib2to3.pgen2.parse import ParseError from lib2to3.pygram import python_symbols from lib2to3.pytree import Leaf, Node from lib2to3.refactor import _detect_future_features added_newline = code and not code.endswith('\n') if added_newline: code += '\n' # Selects the appropriate grammar depending on the usage of # "print_function" future feature. future_features = _detect_future_features(code) if 'print_function' in future_features: grammar = pygram.python_grammar_no_print_statement else: grammar = pygram.python_grammar try: drv = driver.Driver(grammar, pytree.convert) result = drv.parse_string(code, True) except ParseError as e: reraise(e, "Had problems parsing:\n%s\n" % cls._QuotedBlock(code)) # Always return a Node, not a Leaf. if isinstance(result, Leaf): result = Node(python_symbols.file_input, [result]) # Remove AST-leaf for the added newline. if added_newline: last_leaf = _GetLastLeaf(result) if not (last_leaf.type == 0 and last_leaf.value == ''): if last_leaf.prefix: last_leaf.prefix = last_leaf.prefix[:-1] else: last_leaf.remove() return result
def fix_submod_import(self, imported, name, node): u""" Accepts a list of NAME leafs, a name string, and a node node is given as an argument to BaseFix.transform() NAME leafs come from an import_as_names node (the children) name string is the base name found in node. """ submods = [] missed = [] for attr in imported: dotted = u'.'.join((name, attr.value)) if dotted in MAPPING: # get the replacement module to_repl = MAPPING[dotted] if u'.' not in to_repl: # it's a simple name, so use a simple replacement. _import = NameImport(Name(to_repl, prefix=u" "), attr.value) submods.append(_import) elif attr.type == token.NAME: missed.append(attr.clone()) if not submods: return parent = node.parent node.replace(submods[0]) if len(submods) > 1: start = submods.pop(0) prev = start for submod in submods: parent.append_child(submod) if missed: self.warning( node, u"Imported names not known to 3to2 to be part of the package %s. Leaving those alone... high probability that this code will be incorrect." % (name)) children = [ Name(u"from"), Name(name, prefix=u" "), Name(u"import", prefix=u" "), Node(syms.import_as_names, missed) ] orig_stripped = Node(syms.import_from, children) parent.append_child(Newline()) parent.append_child(orig_stripped)
def transform(self, node, results): args = [Leaf(1, 'while')] args += [Leaf(7, '(')] args += [n.clone() for n in results["test_content"]] args += [Leaf(8, ')'),Leaf(token.LBRACE, '{')] args += [n.clone() for n in results["content"]] new = Node(node.type,args) new.prefix = node.prefix return new
def _c_keyword(kwarg): assert kwarg.arg return Node( syms.argument, [ Leaf(token.NAME, kwarg.arg), new(_eq, prefix=''), convert_annotation(kwarg.value), ], )
def add_end_part(end, file, parent, loc): if isNone(end): return if end.type == token.STRING and end.value in ("' '", '" "', "u' '", 'u" "', "b' '", 'b" "'): return if file is None: touch_import(None, "sys", parent) file = Node(syms.power, [Name("sys"), Node(syms.trailer, [Dot(), Name("stdout")])]) end_part = Node(syms.power, [ file, Node(syms.trailer, [Dot(), Name("write")]), Node(syms.trailer, [LParen(), end, RParen()]) ]) end_part.prefix = " " parent.insert_child(loc, Leaf(token.SEMI, ";")) parent.insert_child(loc + 1, end_part)
def make_import(*names, from_module=None): assert names imports = [] if from_module: statement = syms.import_from container = syms.import_as_names single = syms.import_as_name result = [ Leaf(token.NAME, 'from'), Leaf(token.NAME, from_module, prefix=' '), Leaf(token.NAME, 'import', prefix=' '), ] else: statement = syms.import_name container = syms.dotted_as_names single = syms.dotted_as_name result = [Leaf(token.NAME, 'import')] for alias in names: name = Leaf(token.NAME, alias.name, prefix=' ') if alias.asname: _as = Leaf(token.NAME, 'as', prefix=' ') asname = Leaf(token.NAME, alias.asname, prefix=' ') imports.append(Node(single, [name, _as, asname])) else: imports.append(name) if len(imports) == 1: result.append(imports[0]) else: imports_and_commas = [] for imp in imports[:-1]: imports_and_commas.append(imp) imports_and_commas.append(Leaf(token.COMMA, ',')) imports_and_commas.append(imports[-1]) result.append(Node(container, imports_and_commas)) return Node( syms.simple_stmt, [ Node(statement, result), Leaf(token.NEWLINE, '\n'), # FIXME: \r\n? ], )
def transform(self, node, results): args = [] as_list = [[], []] for i in node.children: if i.type == 11: args.append(Leaf(token.LBRACE, '{')) elif i.type == syms.suite: #as 语句 for j in range(len(as_list[0])): args.append(i.children[0].clone()) args.append(i.children[1].clone()) args.append(Node(syms.simple_stmt,[\ Leaf(1, 'auto'), as_list[0][j].clone(), Leaf(22, '='), as_list[1][j].clone(), Leaf(1, ';'),])) as_list[0] = [] as_list[1] = [] args.append(i.clone()) args.append(Leaf(token.RBRACE, '}')) args.append(Leaf(4, '\r\n')) args.append(i.children[-1].clone()) elif i.type == syms.except_clause: except_args = [Leaf(1, 'catch'),\ Leaf(7, '(')] for j in i.children[1:]: #as 语句 if (j.type == 1) & (j.value == 'as'): as_list[0].append(except_args[-1].clone()) elif len(as_list[0]) != len(as_list[1]): as_list[1].append(j.clone()) else: except_args.append(j.clone()) except_args.append(Leaf(8, ')')) args.append( Node(syms.except_clause, except_args, prefix=i.prefix)) else: args.append(i.clone()) new = Node(node.type, args) new.prefix = node.prefix return new
def add_globals(self, node): """Add required globals to the root of node. Idempotent.""" if self.added_pyi_globals: return # TODO: get rid of this -- added to prevent adding .parsed_pyi.top_lines every time # we annotate a different function in the same file, but can break when we run the tool # twice on the same file. Have to do something like what touch_import does. self.added_pyi_globals = True imports, top_lines = self.parsed_pyi.imports, self.parsed_pyi.top_lines # Copy imports if not already present for pkg, names in imports: if names is None: # TODO: do ourselves, touch_import puts stuff above license headers touch_import(None, pkg, node) # == 'import pkg' else: for name in names: touch_import(pkg, name, node) root = find_root(node) import_idx = [ idx for idx, node in enumerate(root.children) if self.import_pattern.match(node) ] if import_idx: future_insert_pos = import_idx[0] top_insert_pos = import_idx[-1] + 1 else: future_insert_pos = top_insert_pos = 0 # first string (normally docstring) for idx, node in enumerate(root.children): if (node.type == syms.simple_stmt and node.children and node.children[0].type == token.STRING): future_insert_pos = top_insert_pos = idx + 1 break top_lines = '\n'.join(top_lines) top_lines = Util.parse_string(top_lines) # strips some newlines for offset, node in enumerate(top_lines.children[:-1]): root.insert_child(top_insert_pos + offset, node) # touch_import doesn't do proper order for __future__ pkg = '__future__' future_imports = [ n for n in self.future_imports if not does_tree_import(pkg, n, root) ] for offset, name in enumerate(future_imports): node = FromImport(pkg, [Leaf(token.NAME, name, prefix=" ")]) node = Node(syms.simple_stmt, [node, Newline()]) root.insert_child(future_insert_pos + offset, node)
def NameImport(package, as_name=None, prefix=None): """ Accepts a package (Name node), name to import it as (string), and optional prefix and returns a node: import <package> [as <as_name>] """ if prefix is None: prefix = u"" children = [Name(u"import", prefix=prefix), package] if as_name is not None: children.extend([Name(u"as", prefix=u" "), Name(as_name, prefix=u" ")]) return Node(syms.import_name, children)
def parse_string(cls, text): """Use lib2to3 to parse text into a Node.""" text = text.strip() if not text: # cls.driver.parse_string just returns the ENDMARKER Leaf, wrap in # a Node for consistency return Node(syms.file_input, [Leaf(token.ENDMARKER, '')]) # workaround: parsing text without trailing '\n' throws exception text += '\n' return cls.driver.parse_string(text)
def transform(self, node, results): args = [Leaf(1, 'for')] args += [Leaf(7, '(')] args += [Leaf(1, 'auto')] #c++11 args += [n.clone() for n in results["in_before"]] args += [Leaf(token.COLON, ':')] args += [n.clone() for n in results["in_after"]] args += [Leaf(8, ')'), Leaf(token.LBRACE, '{')] args += [n.clone() for n in results["content"]] new = Node(node.type, args) new.prefix = node.prefix return new
def _c_call(call): contents = [convert_annotation(arg) for arg in call.args] contents.extend(convert_annotation(kwarg) for kwarg in call.keywords) for index in range(len(contents) - 1, 0, -1): contents[index].prefix = " " contents.insert(index, new(_comma)) call_args = [ new(_lpar), new(_rpar), ] if contents: call_args.insert(1, Node(syms.arglist, contents)) return Node( syms.power, [convert_annotation(call.func), Node( syms.trailer, call_args, )], )
def create_import(import_desc): """Create an AST representing import statement from given description. >>> regenerate(create_import("unittest")) 'import unittest\\n' >>> regenerate(create_import(("nose", "SkipTest"))) 'from nose import SkipTest\\n' """ if isinstance(import_desc, tuple): package, name = import_desc return Node(syms.import_from, [Leaf(token.NAME, 'from'), Leaf(token.NAME, package, prefix=" "), Leaf(token.NAME, 'import', prefix=" "), Leaf(token.NAME, name, prefix=" "), Newline()]) else: return Node(syms.import_name, [Leaf(token.NAME, 'import'), Leaf(token.NAME, import_desc, prefix=" "), Newline()])
def transform_colon(self, node): node_copy = node.clone() # Strip any whitespace that could have been there node_copy.prefix = node_copy.prefix.lstrip() old_depth = find_indentation(node) new_indent = '%s%s' % ((' ' * 4), old_depth) new_node = Node(symbols.suite, [Leaf(token.NEWLINE, '\n'), Leaf(token .INDENT, new_indent), node_copy, Leaf(token.DEDENT, '')]) node.replace(new_node) node.changed() # Replace node with new_node in case semi return node_copy