def _nai_alias(alias, node): # Comments below show example imports that match the rule. name = Leaf(token.NAME, alias.name) if not alias.asname or alias.asname == alias.name: # import hay, x, stack # from field import hay, s, stack if node.type in (syms.dotted_as_names, syms.import_as_names): return name in node.children # import x as x # from field import x as x if node.type in (syms.dotted_as_name, syms.import_as_name): return [name, _as, name] == node.children # import x return node == name asname = Leaf(token.NAME, alias.asname) dotted_as_name = Node(syms.dotted_as_name, [name, _as, asname]) # import hay as stack, x as y if node.type == syms.dotted_as_names: return dotted_as_name in node.children import_as_name = Node(syms.import_as_name, [name, _as, asname]) # from field import hay as stack, x as y if node.type == syms.import_as_names: return import_as_name in node.children # import x as y # from field import x as y return node in (dotted_as_name, import_as_name)
def transform(self, node, results): assert results # Not sure if this check is actually needed, and having it means # we can't remove these imports. Oh well. if not does_tree_import('__future__', 'print_function', node): return assert node.children[0] == Name(u"print") args = node.children[1:] if len(args) != 1: raise RuntimeError('I didn\'t expect this') args_node = args[0].clone() if args_node: if args_node.children[0] == Leaf(token.LPAR, '(') \ and args_node.children[-1] == Leaf(token.RPAR, ')'): args_node.children[0].remove() args_node.children[-1].remove() args_node.prefix = u"" log_call_args = args_node.children if len(log_call_args) == 1 and log_call_args[0].type == syms.arglist: log_call_args = log_call_args[0].children if any(node.type == syms.argument for node in log_call_args): raise RuntimeError( 'Why are there kwargs in here!? How do we handle them? %s', log_call_args) return self._create_logging_call(log_call_args, node)
def add_sep_part(sep, pos, lst): if sep is not None and not isNone(sep) and \ not (sep.type == token.STRING and sep.value in ("' '", '" "')): temp = [] for arg in pos: temp.append(_unicode(arg.clone())) if sys.version_info >= (2, 6): warnings.warn( "Calling unicode() on what may be a bytes object") temp.append(Comma()) del temp[-1] sep = sep.clone() sep.prefix = " " args = Node(syms.listmaker, temp) new_list = Node(syms.atom, [Leaf(token.LSQB, "["), args, Leaf(token.RSQB, "]")]) join_arg = Node(syms.trailer, [LParen(), new_list, RParen()]) sep_join = Node(syms.power, [sep, Node(syms.trailer, [Dot(), Name("join")])]) lst.append(sep_join) lst.append(join_arg) else: if pos: pos[0].prefix = " " for arg in pos: lst.append(arg.clone()) lst.append(Comma()) del lst[-1]
def lazy_aliasing() -> None: # We should find the first place where the alias is used and put it # right above. This way we don't need to look at the value at all. _, prefix = get_offset_and_prefix(body, skip_assignments=True) name_node = Leaf(token.NAME, name) for _offset, stmt in enumerate(body.children): if name_used_in_node(stmt, name_node): break else: _offset = -1 body.children.insert( _offset, Node( syms.simple_stmt, [ Node( syms.expr_stmt, [ Leaf(token.NAME, name), new(_eq), value, ], ), new(_newline), ], prefix=prefix.lstrip('\n'), ), )
def add_py2_annot(self, argtypes, restype, node, results): children = results['suite'][0].children # Insert '# type: {annot}' comment. # For reference, see lib2to3/fixes/fix_tuple_params.py in stdlib. if len(children) >= 1 and children[0].type != token.NEWLINE: # one liner function if children[0].prefix.strip() == '': children[0].prefix = '' children.insert(0, Leaf(token.NEWLINE, '\n')) children.insert( 1, Leaf(token.INDENT, find_indentation(node) + ' ')) children.append(Leaf(token.DEDENT, '')) if len(children) >= 2 and children[1].type == token.INDENT: degen_str = '(...) -> %s' % restype short_str = '(%s) -> %s' % (', '.join(argtypes), restype) if (len(short_str) > 64 or len(argtypes) > 5) and len(short_str) > len(degen_str): self.insert_long_form(node, results, argtypes) annot_str = degen_str else: annot_str = short_str children[1].prefix = '%s# type: %s\n%s' % ( children[1].value, annot_str, children[1].prefix) children[1].changed() else: self.log_message( "%s:%d: cannot insert annotation for one-line function" % (self.filename, node.get_lineno()))
def touch_import_top(package, name_to_import, node): """Works like `does_tree_import` but adds an import statement at the top if it was not imported (but below any __future__ imports). Calling this multiple times adds them in reverse order. Based on lib2to3.fixer_util.touch_import() """ root = find_root(node) if does_tree_import(package, name_to_import, root): return # Look for __future__ imports and insert below them found = False for name in [ 'absolute_import', 'division', 'print_function', 'unicode_literals' ]: if does_tree_import('__future__', name, root): found = True break if found: # At least one __future__ import. We want to loop until we've seen them # all. start, end = None, None for idx, node in enumerate(root.children): if check_future_import(node): start = idx # Start looping idx2 = start while node: node = node.next_sibling idx2 += 1 if not check_future_import(node): end = idx2 break break assert start is not None assert end is not None insert_pos = end else: # No __future__ imports for idx, node in enumerate(root.children): if node.type == syms.simple_stmt: # and node.children and node.children[0].type == token.STRING): break insert_pos = idx if package is None: import_ = Node(syms.import_name, [ Leaf(token.NAME, u"import"), Leaf(token.NAME, name_to_import, prefix=u" ") ]) else: import_ = FromImport(package, [Leaf(token.NAME, name_to_import, prefix=u" ")]) children = [import_, Newline()] root.insert_child(insert_pos, Node(syms.simple_stmt, children))
def transform(self, node, results): args = [Leaf(1, 'while')] args += [Leaf(7, '(')] args += [n.clone() for n in results["test_content"]] args += [Leaf(8, ')'),Leaf(token.LBRACE, '{')] args += [n.clone() for n in results["content"]] new = Node(node.type,args) new.prefix = node.prefix return new
def _get_list(self, *nodes): """A List node, filled""" lbrace = Leaf(token.LBRACE, u"[") lbrace.prefix = u" " if len(nodes) > 0: nodes[0].prefix = u"" return Node(self.syms.trailer, [lbrace] + [node.clone() for node in nodes] + [Leaf(token.RBRACE, u"]")])
def form_eqs(node1,node2): result = Node(syms.simple_stmt,[\ Node(syms.expr_stmt,[\ node1.clone(),\ Leaf(22, '='),\ node2.clone()]),\ Leaf(4, '\r\n')\ ]) result.changed() return result
def transform(self, node, results): args = [] item_id = 0 while item_id < len(node.children): if (node.children[item_id].__class__.__name__ == "Leaf"): if node.children[item_id].value == ':': #加花括号 args += [ Leaf(token.LBRACE, '{'), node.children[item_id + 1].clone(), Leaf(token.RBRACE, '}'), Leaf(4, '\r\n') ] item_id += 1 #if,elif处理 elif (node.children[item_id].value == 'elif') | (node.children[item_id].value == 'if'): if node.children[item_id].value == 'elif': leaf_temp = Leaf(1, 'if') leaf_temp.prefix = " " args += [Leaf(1, 'else'), leaf_temp] else: args += [node.children[item_id].clone()] args += [Leaf(7, '(')] args += [node.children[item_id + 1].clone()] args += [Leaf(8, ')')] item_id += 1 else: args += [node.children[item_id].clone()] else: print("\nFixerError_if\n") item_id += 1 new = Node(node.type, args) new.prefix = node.prefix return new
def transform_arglist_to_keywords(self, arglist_node, alg_object): """Takes a node that points to argument list and transforms it to all keyword=values @param arglist_node The node that points to the argument list @param alg_object The algorithm object that corresponds to this list """ ordered_props = alg_object.orderedProperties() # Special case where the arglist has no children if len(arglist_node.children) == 0: arglist_node = Node(syms.argument, [Leaf(token.NAME,ordered_props[0]),Leaf(token.EQUAL,"="), Leaf(arglist_node.type,arglist_node.value)]) return arglist_node # Quick check: A 3 arg leaf list with an equals at the 2nd element needs nothing doing if len(arglist_node.children) == 3 and arglist_node.children[1].type == token.EQUAL: return arglist_node # Construct our argument list from the children to make sure we separate out whole comma-separated # sections i.e get embedded lists correct args = [[]] # Each list will be delimited by a comma nargs = 0 index = 0 for node in arglist_node.children: if node.type == token.COMMA: args.append(node) args.append([]) # new arg list index += 2 # include comma nargs += 1 else: args[index].append(node) # Ordered props prop_index = 0 # List has commas so standard enumerate won't do the trick arg_nodes = [] # Holds the final node list for arg_list in args: if isinstance(arg_list, Leaf): # Must be comma from construction above arg_nodes.append(arg_list) else: first = arg_list[0] if not (isinstance(first, Node) and first.type == syms.argument): prop_name = ordered_props[prop_index] children=[Leaf(token.NAME,prop_name),Leaf(token.EQUAL,"=")] children.extend(arg_list) for c in children: c.parent = None # A new node requires all old parent nodes to be None arg_nodes.append(Node(syms.argument, children)) else: for node in arg_list: arg_nodes.append(node) # Move to the next property prop_index += 1 arglist_node.children = arg_nodes return arglist_node
def transform_colon(self, node): node_copy = node.clone() # Strip any whitespace that could have been there node_copy.prefix = node_copy.prefix.lstrip() old_depth = find_indentation(node) new_indent = '%s%s' % ((' ' * 4), old_depth) new_node = Node(symbols.suite, [Leaf(token.NEWLINE, '\n'), Leaf(token .INDENT, new_indent), node_copy, Leaf(token.DEDENT, '')]) node.replace(new_node) node.changed() # Replace node with new_node in case semi return node_copy
def transform(self, node, results): self.count += 1 if 'arg_2' in results: logger.debug("found 2 [%s %s]" % (results['arg_1'], results['arg_2'])) error_no = results['arg_1'] error_no.replace(Leaf(type=2, value=self.count)) else: logger.debug("found 1 [%s]" % results['arg_1']) siblings_list = results['arg_1'].parent.children siblings_list.insert(1, Leaf(type=2, value=self.count)) siblings_list.insert(2, Comma()) siblings_list[3].prefix = " " return node
def transform(self, node, results): args = [] if results['first']: args.append(Node(syms.simple_stmt,[\ Leaf(1, 'TODO_PyObject'), results['name'].clone(), Leaf(1, ';'), Leaf(4, '\r\n'),])) if results['out'][0].type==syms.trailer: args.append(deal_cout(results['out'][0].children[1])) args.append(deal_cin(results['name'])) result = Node(syms.testlist1,args) result.prefix = node.prefix return result
def deal_term(node): from lib2to3.fixer_util import Call args = [] args += get_atom(node.children[0]) for i in range(1, len(node.children)): if i % 2 == 1: continue args += get_atom(node.children[i]) temp = len(args) - 1 for i in range(temp): args.insert(2 * i + 1, Leaf(12, ',')) return Node(syms.simple_stmt,[\ Call(Leaf(1,'printf'),args), Leaf(1, ';'), Leaf(4, '\r\n'),])
def test(self): kids = [None, [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 2), Leaf(token.NUMBER, 3)], [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 3), Leaf(token.NUMBER, 2), Leaf(token.NUMBER, 4)], [Leaf(token.STRING, "b"), Leaf(token.STRING, "j", prefix=" ")] ] self.assertStr(self._Call("A"), "A()") self.assertStr(self._Call("b", kids[1]), "b(1,2,3)") self.assertStr(self._Call("a.b().c", kids[2]), "a.b().c(1,3,2,4)") self.assertStr(self._Call("d", kids[3], prefix=" "), " d(b, j)")
def future_import(feature, node): """ This seems to work """ root = find_root(node) if does_tree_import(u"__future__", feature, node): return for idx, node in enumerate(root.children): if node.type == syms.simple_stmt and \ len(node.children) > 0 and node.children[0].type == token.STRING: # skip over docstring continue names = check_future_import(node) if not names: # not a future statement; need to insert before this break if feature in names: # already imported return import_ = FromImport(u'__future__', [Leaf(token.NAME, feature, prefix=" ")]) children = [import_, Newline()] root.insert_child(idx, Node(syms.simple_stmt, children))
def _r_functiondef(fun, node, flags): assert node.type in (syms.file_input, syms.suite) name = Leaf(token.NAME, fun.name) pyi_decorators = decorator_names(fun.decorator_list) pyi_method_decorators = list( filter(is_builtin_method_decorator, pyi_decorators)) or ["instancemethod"] is_method = (node.parent is not None and node.parent.type == syms.classdef and "staticmethod" not in pyi_method_decorators) args, returns = get_function_signature(fun, is_method=is_method) for child in flatten_some(node.children): decorators = None if child.type == syms.decorated: # skip decorators decorators = child.children[0] child = child.children[1] if child.type in (syms.async_stmt, syms.async_funcdef): # async def in 3.5 and 3.6 child = child.children[1] if child.type != syms.funcdef: continue offset = 1 if child.children[offset] == name: lineno = child.get_lineno() column = 1 if decorators: src_decorators = decorator_names(decorators) src_method_decorators = list( filter(is_builtin_method_decorator, src_decorators)) or ["instancemethod"] if pyi_method_decorators != src_method_decorators: raise ValueError( f"Incompatible method kind for {fun.name!r}: " + f"{lineno}:{column}: Expected: " + f"{pyi_method_decorators[0]}, actual: " + f"{src_method_decorators[0]}") is_method = "staticmethod" not in pyi_decorators try: annotate_parameters(child.children[offset + 1], args, is_method=is_method, flags=flags) annotate_return(child.children, returns, offset + 2, flags) reapply(fun.body, child.children[-1], flags) remove_function_signature_type_comment(child.children[-1]) except ValueError as ve: raise ValueError( f"Annotation problem in function {name.value!r}: " + f"{lineno}:{column}: {ve}") from ve break else: raise ValueError(f"Function {name.value!r} not found in source.") return []
def recur_type(node,varlist=[[]]): #寻找函数参数并添加进varlist def function_recur(node,varlist): for i in node.children: if i.type==syms.parameters: for j in i.children: if j.type==syms.typedargslist: for k in j.children: if(k.type==1): varlist[-1].append(k.value) elif j.type==1: varlist[-1].append(j.value) recur_type(i,varlist) if(len(node.children)==0): return node for i in node.children: #递归 if(i.type!=syms.expr_stmt): if(i.type==syms.funcdef): function_recur(i,varlist+[[]]) elif(i.type==syms.classdef): recur_type(i,varlist+[[]]) elif i.was_changed: continue #是下级递归添加的 elif i.type==syms.global_stmt: for j in i.children[1:]: if(j.type==1): varlist[-1].append(j.value) else: recur_type(i,varlist) #处理+=问题 elif Leaf(22, '=') not in i.children: continue #处理赋值语句 else: eqs_dict = eqs_deal(i) #不相等意味着存在多返回值函数 if len(eqs_dict['start'])!=len(eqs_dict['end']): i.prefix+=" warning: multiple return value include " continue #start end处理 #TODO:prefix 测试 parent_id = i.parent.parent.children.index(i.parent) for id in range(len(eqs_dict['start'])): node_now = form_eqs(eqs_dict['start'][id],eqs_dict['end'][id]) if judge_first(eqs_dict['start'][id],varlist[-1],node_now): varlist[-1].append(eqs_dict['start'][id].value) if(id==0): node_now.prefix = i.prefix node_now.children[0].prefix = i.children[0].prefix i.parent.parent.insert_child(parent_id+id,node_now) parent_id+=len(eqs_dict['start']) #mid处理 for id in range(len(eqs_dict['mid'])): node_now = form_mideqs(eqs_dict['mid'][id],eqs_dict['end']) if judge_first(eqs_dict['mid'][id],varlist[-1],node_now): varlist[-1].append(eqs_dict['mid'][id].value) i.parent.parent.insert_child(parent_id+id,node_now) i.parent.remove()
def fix_not_a_in_b(node: LN, capture: Capture, filename: Filename): capture["element"].parent = None capture["collection"].parent = None new_comparison = Node( comparison, [ capture["element"], Node(comp_op, [Leaf(1, "not", prefix=" "), Leaf(1, "in", prefix=" ")]), capture["collection"], ], ) new_comparison.parent = node.parent return new_comparison
def future_import2(feature, node): """ An alternative to future_import() which might not work ... """ root = find_root(node) if does_tree_import(u"__future__", feature, node): return insert_pos = 0 for idx, node in enumerate(root.children): if node.type == syms.simple_stmt and node.children and \ node.children[0].type == token.STRING: insert_pos = idx + 1 break for thing_after in root.children[insert_pos:]: if thing_after.type == token.NEWLINE: insert_pos += 1 continue prefix = thing_after.prefix thing_after.prefix = u"" break else: prefix = u"" import_ = FromImport(u"__future__", [Leaf(token.NAME, feature, prefix=u" ")]) children = [import_, Newline()] root.insert_child(insert_pos, Node(syms.simple_stmt, children, prefix=prefix))
def __insertImport(self, module, offset): if self.__coreImport is None: return importLine = Node(syms.simple_stmt, [ Node(syms.import_name, [ Leaf(token.NAME, u"import"), Leaf(token.NAME, module, prefix=" ") ]), Newline() ]) self.__coreImport.parent.insert_child( self.__coreImport.parent.children.index(self.__coreImport) + offset, importLine)
def insert_comment_annotation(self, src_sig, pyi_sig): """Insert function annotation as a comment string.""" inserted_types = [] str_arg_types = [] for i, (arg_sig, pyi_arg_sig) in enumerate( zip(src_sig.arg_sigs, pyi_sig.arg_sigs)): is_first = (i == 0) new_type = clean_clone(pyi_arg_sig.arg_type, True) if new_type: new_type_str = str(new_type).strip() inserted_types.append(new_type) elif self.infer_should_annotate(src_sig, arg_sig, is_first): new_type_str = 'Any' else: continue str_arg_types.append(arg_sig.stars + new_type_str) ret_type = pyi_sig.ret_type if ret_type: inserted_types.append(ret_type) else: ret_type = self.infer_ret_type(src_sig) annot = '(' + ', '.join(str_arg_types) + ') -> ' + str( ret_type).strip() if src_sig.try_insert_comment_annotation(annot): if 'Any' in annot: inserted_types.append(Leaf(token.NAME, 'Any')) return inserted_types else: return []
def suitify(parent): """ Turn the stuff after the first colon in parent's children into a suite, if it wasn't already """ for node in parent.children: if node.type == syms.suite: # already in the prefered format, do nothing return # One-liners have no suite node, we have to fake one up for i, node in enumerate(parent.children): if node.type == token.COLON: break else: raise ValueError(u"No class suite and no ':'!") # Move everything into a suite node suite = Node(syms.suite, [ Newline(), Leaf(token.INDENT, indentation(node) + indentation_step(node)) ]) one_node = parent.children[i + 1] one_node.remove() one_node.prefix = u'' suite.append_child(one_node) parent.append_child(suite)
def add_future(node, symbol): root = fixer_util.find_root(node) for idx, node in enumerate(root.children): if node.type == syms.simple_stmt and \ len(node.children) > 0 and node.children[0].type == token.STRING: # skip over docstring continue names = check_future_import(node) if not names: # not a future statement; need to insert before this break if symbol in names: # already imported return import_ = fixer_util.FromImport('__future__', [Leaf(token.NAME, symbol, prefix=" ")]) # Place after any comments or whitespace. (copyright, shebang etc.) import_.prefix = node.prefix node.prefix = '' children = [import_, fixer_util.Newline()] root.insert_child(idx, Node(syms.simple_stmt, children))
def test_is_shebang_comment(self): """ Tests whether the libfuturize.fixer_util.is_shebang_comment() function is working """ node = FromImport(u'math', [Leaf(token.NAME, u'cos', prefix=" ")]) node.prefix = u'#!/usr/bin/env python\n' self.assertTrue(is_shebang_comment(node))
def insert_long_form(self, node, results, argtypes): # type: (Node, Dict[str, Any], List[str]) -> None argtypes = list(argtypes) # We destroy it args = results['args'] if isinstance(args, Node): children = args.children elif isinstance(args, Leaf): children = [args] else: children = [] # Interpret children according to the following grammar: # (('*'|'**')? NAME ['=' expr] ','?)* flag = False # Set when the next leaf should get a type prefix indent = '' # Will be set by the first child def set_prefix(child): if argtypes: arg = argtypes.pop(0).lstrip('*') else: arg = 'Any' # Somehow there aren't enough args if not arg: # Skip self (look for 'check_self' below) prefix = child.prefix.rstrip() else: prefix = ' # type: ' + arg old_prefix = child.prefix.strip() if old_prefix: assert old_prefix.startswith('#') prefix += ' ' + old_prefix child.prefix = prefix + '\n' + indent check_self = self.is_method(node) for child in children: if isinstance(child, Leaf): if check_self and child.type == token.NAME: check_self = False if child.value in ('self', 'cls'): argtypes.insert(0, '') if not indent: indent = ' ' * child.column if child.value == ',': flag = True elif flag: set_prefix(child) flag = False need_comma = len(children) >= 1 and children[-1].type != token.COMMA if need_comma and len(children) >= 2: if (children[-1].type == token.NAME and (children[-2].type in (token.STAR, token.DOUBLESTAR))): need_comma = False if need_comma: children.append(Leaf(token.COMMA, u",")) # Find the ')' and insert a prefix before it too. parameters = args.parent close_paren = parameters.children[-1] assert close_paren.type == token.RPAR, close_paren set_prefix(close_paren) assert not argtypes, argtypes
def walk_dedent_tree(node, children, indent): force_split_next = False for item in children: prev = item.prev_sibling if not prev: if isinstance(item, Leaf) and six.text_type(item).startswith("\n"): prev = node.prev_sibling next = node.next_sibling final_length = 0 if prev and "\n" not in six.text_type(node).strip(): final_length = prev.column + len( six.text_type(node).strip()) + 3 item.replace(Leaf( item.type, item.value, prefix=' ', )) if final_length and final_length > MAX_LINE_LENGTH: # tell next call to walk_dedent_tree_node that we need # different stringformat tactic force_split_next = True elif isinstance(item, Node): if node.type == syms.power: for subitem in item.children[1:]: walk_dedent_power_node(subitem, subitem.children, indent) else: for subitem in item.children[1:]: walk_dedent_tree_node(subitem, subitem.children, indent, force_split_next) force_split_next = False
def transform(self, node, results): syms = self.syms exc, val, trc = (results[u"exc"], results[u"val"], results[u"trc"]) val = val[0] if val else Leaf(token.NAME, u"None") val.prefix = trc.prefix = u" " kids = [exc.clone(), Comma(), val.clone(), Comma(), trc.clone()] args = results[u"args"] args.children = kids
def transform_semi(self, node): for child in node.children: if child.type == token.SEMI: # Strip any whitespace from the next sibling if (child.next_sibling.prefix != child.next_sibling.prefix.lstrip()): child.next_sibling.prefix = ( child.next_sibling.prefix.lstrip()) child.next_sibling.changed() # Replace the semi with a newline old_depth = find_indentation(child) child.replace( [Leaf(token.NEWLINE, '\n'), Leaf(token.INDENT, old_depth)]) child.changed() return node
def finish_tree(self, tree, filename): if not self._names: return names = [Leaf(token.LBRACE, "[", prefix=" "), Newline()] for name in self._names: names.append(String('"' + name + '"', prefix=" ")) names.append(Comma()) names.append(Newline()) names.append(Leaf(token.LBRACE, "]", prefix="")) tree.append_child(Assign(Name("__all__"), names)) tree.append_child(Newline()) super(FixAllAttribute, self).finish_tree(tree, filename)