def finish_tree(self, tree, filename):
if self.found_future_import:
return
if not isinstance(tree, pytree.Node):
# Empty files (usually __init__.py) show up as a single Leaf
# instead of a Node, so leave them alone
return
first_stmt = tree.children[0]
if is_docstring(first_stmt):
# Skip a line and add the import after the docstring
tree.insert_child(1, Newline())
pos = 2
elif first_stmt.prefix:
# No docstring, but an initial comment (perhaps a #! line).
# Transfer the initial comment to a new blank line.
newline = Newline()
newline.prefix = first_stmt.prefix
first_stmt.prefix = ""
tree.insert_child(0, newline)
pos = 1
else:
# No comments or docstring, just insert at the start
pos = 0
tree.insert_child(pos, self.new_future_import(None))
tree.insert_child(pos + 1, Newline()) # terminates the import stmt
def RaisesRegexOp(context, designator, exceptionClass, expected_regex, indent,
kws, arglist, node):
expected_regex.prefix = ""
arglist = [a.clone() for a in arglist.children]
del arglist[2:4] # remove pattern and comma
arglist = Node(syms.arglist, arglist)
with_stmt = RaisesOp(context, exceptionClass, indent, kws, arglist, node)
with_stmt.insert_child(2, Name('as', prefix=" "))
with_stmt.insert_child(3, Name(designator, prefix=" "))
# if this is already part of a with statement we need to insert re.search
# after the last leaf with content
if node.parent.type == syms.with_stmt:
parent_with = node.parent
for leaf in reversed(list(parent_with.leaves())):
if leaf.value.strip():
break
i = leaf.parent.children.index(leaf)
leaf.parent.insert_child(i + 1, Newline())
leaf.parent.insert_child(
i + 2,
Name('assert %s.match(%s)' % (designator, expected_regex),
prefix=indent))
return with_stmt
else:
return Node(syms.suite, [
with_stmt,
Newline(),
Name('assert %s.match(%s)' % (designator, expected_regex),
prefix=indent)
])
def transform(self, node, results):
singleton = results.get("one_dec")
classdef = results["cls"]
decs = [results["one_dec"]
] if results.get("one_dec") is not None else results["decs"]
dec_strings = [str(dec).strip()[1:] for dec in decs]
assign = ""
for dec in dec_strings:
assign += dec
assign += "("
assign += results["name"].value
for dec in dec_strings:
assign += ")"
assign = String(results["name"].value + " = " + assign)
assign_statement = Node(
syms.simple_stmt, [assign, Newline(), Newline()])
prefix = None
for dec in decs:
if prefix is None:
prefix = dec.prefix
dec.remove()
classdef.prefix = prefix
i = indentation(node)
pos = node.children.index(classdef) + 1
if classdef.children[-1].children[-1].type == token.DEDENT:
del classdef.children[-1].children[-1]
node.insert_child(pos, Leaf(token.INDENT, i))
node.insert_child(pos, assign_statement)
node.insert_child(pos, Leaf(token.INDENT, i))
def finish_tree(self, tree, filename):
if self.found_future_import:
return
if not isinstance(tree, pytree.Node):
# Empty files (usually __init__.py) show up as a single Leaf
# instead of a Node, so leave them alone
return
first_stmt = tree.children[0]
if is_docstring(first_stmt):
# Skip a line and add the import after the docstring
tree.insert_child(1, Newline())
pos = 2
elif first_stmt.prefix:
# No docstring, but an initial comment (perhaps a #! line).
# Transfer the initial comment to a new blank line.
newline = Newline()
newline.prefix = first_stmt.prefix
first_stmt.prefix = ""
tree.insert_child(0, newline)
pos = 1
else:
# No comments or docstring, just insert at the start
pos = 0
tree.insert_child(pos, self.new_future_import(None))
tree.insert_child(pos+1, Newline()) # terminates the import stmt
def _CreateCode(indent, package, symbols, comment):
"""
Create code:
from <package> import <symbols> # <comment>
"""
# children: the children nodes for the final from-import statement
children = [
Name('from', prefix=' ' * indent),
Name(package, prefix=' '),
Name('import', prefix=' '),
]
# name_leaf: list of leaf nodes with the symbols to import
name_leafs = []
symbols = sorted(symbols)
for i, i_symbol in enumerate(symbols):
prefix = ' ' if i == 0 else ', '
name_leafs.append(i_symbol.CreateNameNode(prefix))
# nodes_wrap: if true, we need to wrap the import statement
nodes_wrap = False
line_len = 0
line_len += six.moves.reduce(lambda x, y: x + y,
map(len, map(str, children)), 0)
line_len += six.moves.reduce(lambda x, y: x + y,
map(len, map(str, name_leafs)), 0)
if line_len > page_width:
# Add parenthesis around the "from" names
name_leafs[0].prefix = ''
name_leafs.insert(0, Name('(', prefix=' '))
name_leafs.append(Name(')'))
nodes_wrap = True
# Adds the name_leafs to the children list
children += [
Node(pygram.python_symbols.import_as_names, name_leafs)
]
# from_import: the final node for the import statement
from_import = Node(pygram.python_symbols.import_from, children)
# result: a simple-statement node with the import statement and
# EOL.
new_line = Newline()
new_line.prefix = comment
result = Node(
pygram.python_symbols.simple_stmt,
children=[
from_import,
new_line,
],
)
# Wrap nodes if necessary (see nodes_wrap above)
if nodes_wrap:
ImportBlock.TextWrapForNode(result, page_width, indent)
return result
def _CreateCode(indent, package, symbols, comment):
'''
Create code:
from <package> import <symbols> # <comment>
'''
# children: the children nodes for the final from-import statement
children = [
Name('from', prefix=' ' * indent),
Name(package, prefix=' '),
Name('import', prefix=' '),
]
# name_leaf: list of leaf nodes with the symbols to import
name_leafs = []
symbols = sorted(symbols)
for i, i_symbol in enumerate(symbols):
prefix = ' ' if i == 0 else ', '
name_leafs.append(i_symbol.CreateNameNode(prefix))
# nodes_wrap: if true, we need to wrap the import statement
nodes_wrap = False
line_len = 0
line_len += reduce(lambda x, y:x + y, map(len, map(str, children)), 0)
line_len += reduce(lambda x, y:x + y, map(len, map(str, name_leafs)), 0)
if line_len > page_width:
# Add parenthesis around the "from" names
name_leafs[0].prefix = ''
name_leafs.insert(0, Name('(', prefix=' '))
name_leafs.append(Name(')'))
nodes_wrap = True
# Adds the name_leafs to the children list
children += [Node(pygram.python_symbols.import_as_names, name_leafs)]
# from_import: the final node for the import statement
from_import = Node(pygram.python_symbols.import_from, children)
# result: a simple-statement node with the import statement and EOL.
new_line = Newline()
new_line.prefix = comment
result = Node(
pygram.python_symbols.simple_stmt,
children=[
from_import,
new_line,
],
)
# Wrap nodes if necessary (see nodes_wrap above)
if nodes_wrap:
ImportBlock.TextWrapForNode(result, page_width, indent)
return result
def transform(self, node, results):
params_rawlist = results[u"params"]
for i, item in enumerate(params_rawlist):
if item.type == token.STAR:
params_rawlist = params_rawlist[i:]
break
else:
return
# params is guaranteed to be a list starting with *.
# if fixing is needed, there will be at least 3 items in this list:
# [STAR, COMMA, NAME] is the minimum that we need to worry about.
new_kwargs = needs_fixing(params_rawlist)
# new_kwargs is the name of the kwargs dictionary.
if not new_kwargs:
return
suitify(node)
# At this point, params_rawlist is guaranteed to be a list
# beginning with a star that includes at least one keyword-only param
# e.g., [STAR, NAME, COMMA, NAME, COMMA, DOUBLESTAR, NAME] or
# [STAR, COMMA, NAME], or [STAR, COMMA, NAME, COMMA, DOUBLESTAR, NAME]
# Anatomy of a funcdef: ['def', 'name', parameters, ':', suite]
# Anatomy of that suite: [NEWLINE, INDENT, first_stmt, all_other_stmts]
# We need to insert our new stuff before the first_stmt and change the
# first_stmt's prefix.
suite = node.children[4]
first_stmt = suite.children[2]
ident = indentation(first_stmt)
for name, default_value in gen_params(params_rawlist):
if default_value is None:
suite.insert_child(2, Newline())
suite.insert_child(2, String(_assign_template %{u'name':name, u'kwargs':new_kwargs}, prefix=ident))
else:
suite.insert_child(2, Newline())
suite.insert_child(2, String(_else_template %{u'name':name, u'default':default_value}, prefix=ident))
suite.insert_child(2, Newline())
suite.insert_child(2, String(_if_template %{u'assign':_assign_template %{u'name':name, u'kwargs':new_kwargs}, u'name':name, u'kwargs':new_kwargs}, prefix=ident))
first_stmt.prefix = ident
suite.children[2].prefix = u""
# Now, we need to fix up the list of params.
must_add_kwargs = remove_params(params_rawlist)
if must_add_kwargs:
arglist = results[u'arglist']
if len(arglist.children) > 0 and arglist.children[-1].type != token.COMMA:
arglist.append_child(Comma())
arglist.append_child(DoubleStar(prefix=u" "))
arglist.append_child(Name(new_kwargs))
def CreateCode(self, indent, page_width):
from lib2to3 import pygram
from lib2to3.fixer_util import Name, Newline
from lib2to3.pytree import Node
assert self.kind == self.KIND_IMPORT_NAME
name_node = self.CreateNameNode()
new_line = Newline()
new_line.prefix = self.comment
result = Node(pygram.python_symbols.simple_stmt,
prefix=' ' * indent,
children=[Name('import'), name_node, new_line])
return [result]
def CreateCode(self, indent, page_width):
from lib2to3 import pygram
from lib2to3.fixer_util import Name, Newline
from lib2to3.pytree import Node
assert self.kind == self.KIND_IMPORT_NAME
name_node = self.CreateNameNode()
new_line = Newline()
new_line.prefix = self.comment
result = Node(
pygram.python_symbols.simple_stmt,
prefix=' ' * indent,
children=[Name('import'), name_node, new_line]
)
return [result]
def add_global_assignment_after_imports(_name, assignment, node):
"""
Big copy paste + modification from touch_import
"""
root = find_root(node)
if find_binding(_name, root):
return
# figure out where to insert the assignment.
# First try to find the first import and then skip to the last one.
insert_pos = offset = 0
for idx, node in enumerate(root.children):
if not is_import_ish_stmt(node):
continue
for offset, node2 in enumerate(root.children[idx:]):
if not is_import_ish_stmt(node2):
break
insert_pos = idx + offset
break
# if there are no imports where we can insert, find the docstring.
# if that also fails, we stick to the beginning of the file
if insert_pos == 0:
for idx, node in enumerate(root.children):
if (node.type == syms.simple_stmt and node.children
and node.children[0].type == token.STRING):
insert_pos = idx + 1
break
children = [assignment, Newline()]
root.insert_child(insert_pos, Node(syms.simple_stmt, children))
def _new_type_check_with_import(package, name, root, insert_pos):
# type: (Optional[str], str, Node, int) -> None
"""
Inserts a new TYPE_CHECKING block containing a new import statement for package and name
Parameters
-----------
package : Optional[str]
name : str
root : Node
insert_pos : int
"""
# [Grammar]
# if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
type_check_node = Node(syms.if_stmt,
[Leaf(token.NAME, 'if'),
Leaf(token.NAME, 'TYPE_CHECKING', prefix=" "),
Leaf(token.COLON, ':'),
# [Grammar]
# suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
Node(syms.suite, [Leaf(token.NEWLINE, '\n'),
Leaf(token.INDENT, ' '),
Node(syms.simple_stmt,
[_generate_import_node(package, name), Newline()]),
Leaf(token.DEDENT, '')])])
# We can just hardcode the correct insert position since we just created the typing block
root.insert_child(insert_pos, type_check_node)
# Make sure to import TYPE_CHECKING just before using
import_type_checking = [_generate_import_node('typing', 'TYPE_CHECKING'), Newline()]
root.insert_child(insert_pos, Node(syms.simple_stmt, import_type_checking))
def RaisesOp(context, exceptionClass, indent, kws, arglist, node):
with_item = Call(Name(context), [exceptionClass])
with_item.prefix = " "
args = []
arglist = [a.clone() for a in arglist.children[4:]]
if arglist:
arglist[0].prefix = ""
func = None
# :fixme: this uses hardcoded parameter names, which may change
if 'callableObj' in kws:
func = kws['callableObj']
elif 'callable_obj' in kws:
func = kws['callable_obj']
elif kws['args']: # any arguments assigned to `*args`
func = kws['args'][0]
else:
func = None
if func is None:
# Context manager
return Node(syms.with_stmt, [with_item])
if func.type == syms.lambdef:
suite = func.children[-1].clone()
else:
suite = Call(func, arglist)
suite.prefix = indent + (4 * " ")
return Node(
syms.with_stmt,
[Name('with'), with_item,
Name(':'), Newline(), suite])
def transform(self, node, results):
if 'sys_import' in results:
if self.sys_import is None:
self.sys_import = results['sys_import']
return
else:
func = results['func'].clone()
func.prefix = u''
register = pytree.Node(syms.power, Attr(Name(u'atexit'), Name(u'register')))
call = Call(register, [func], node.prefix)
node.replace(call)
if self.sys_import is None:
self.warning(node, "Can't find sys import; Please add an atexit import at the top of your file.")
return
names = self.sys_import.children[1]
if names.type == syms.dotted_as_names:
names.append_child(Comma())
names.append_child(Name(u'atexit', u' '))
else:
containing_stmt = self.sys_import.parent
position = containing_stmt.children.index(self.sys_import)
stmt_container = containing_stmt.parent
new_import = pytree.Node(syms.import_name, [Name(u'import'), Name(u'atexit', u' ')])
new = pytree.Node(syms.simple_stmt, [new_import])
containing_stmt.insert_child(position + 1, Newline())
containing_stmt.insert_child(position + 2, new)
return
def future_import2(feature, node):
"""
An alternative to future_import() which might not work ...
"""
root = find_root(node)
if does_tree_import(u"__future__", feature, node):
return
insert_pos = 0
for idx, node in enumerate(root.children):
if node.type == syms.simple_stmt and node.children and \
node.children[0].type == token.STRING:
insert_pos = idx + 1
break
for thing_after in root.children[insert_pos:]:
if thing_after.type == token.NEWLINE:
insert_pos += 1
continue
prefix = thing_after.prefix
thing_after.prefix = u""
break
else:
prefix = u""
import_ = FromImport(u"__future__",
[Leaf(token.NAME, feature, prefix=u" ")])
children = [import_, Newline()]
root.insert_child(insert_pos,
Node(syms.simple_stmt, children, prefix=prefix))
def future_import(feature, node):
"""
This seems to work
"""
root = find_root(node)
if does_tree_import(u"__future__", feature, node):
return
for idx, node in enumerate(root.children):
if node.type == syms.simple_stmt and \
len(node.children) > 0 and node.children[0].type == token.STRING:
# skip over docstring
continue
names = check_future_import(node)
if not names:
# not a future statement; need to insert before this
break
if feature in names:
# already imported
return
import_ = FromImport(u'__future__',
[Leaf(token.NAME, feature, prefix=" ")])
children = [import_, Newline()]
root.insert_child(idx, Node(syms.simple_stmt, children))
def transform(self, node, results):
meta_results = has_metaclass(node)
if not meta_results: return
for meta in meta_results:
meta.remove()
target = Leaf(token.NAME, u"__metaclass__")
equal = Leaf(token.EQUAL, u"=", prefix=u" ")
# meta is the last item in what was returned by has_metaclass(): name
name = meta
name.prefix = u" "
stmt_node = Node(syms.atom, [target, equal, name])
suitify(node)
for item in node.children:
if item.type == syms.suite:
for stmt in item.children:
if stmt.type == token.INDENT:
# Insert, in reverse order, the statement, a newline,
# and an indent right after the first indented line
loc = item.children.index(stmt) + 1
# Keep consistent indentation form
ident = Leaf(token.INDENT, stmt.value)
item.insert_child(loc, ident)
item.insert_child(loc, Newline())
item.insert_child(loc, stmt_node)
break
def suitify(parent):
"""
Turn the stuff after the first colon in parent's children
into a suite, if it wasn't already
"""
for node in parent.children:
if node.type == syms.suite:
# already in the prefered format, do nothing
return
# One-liners have no suite node, we have to fake one up
for i, node in enumerate(parent.children):
if node.type == token.COLON:
break
else:
raise ValueError(u"No class suite and no ':'!")
# Move everything into a suite node
suite = Node(syms.suite, [
Newline(),
Leaf(token.INDENT,
indentation(node) + indentation_step(node))
])
one_node = parent.children[i + 1]
one_node.remove()
one_node.prefix = u''
suite.append_child(one_node)
parent.append_child(suite)
def RaisesOp(context, exceptionClass, indent, kws, arglist):
with_item = Call(Name(context), [exceptionClass])
with_item.prefix = " "
args = []
arglist = [a.clone() for a in arglist.children[4:]]
if arglist:
arglist[0].prefix=""
func = None
# :fixme: this uses hardcoded parameter names, which may change
if 'callableObj' in kws:
func = kws['callableObj']
elif 'callable_obj' in kws:
func = kws['callable_obj']
elif kws['args']: # any arguments assigned to `*args`
func = kws['args'][0]
else:
raise NotImplementedError('with %s is not implemented' % context)
if func is unittest.case._sentinel:
# with self.assertRaises(SomeException):
return Node(syms.with_stmt,
[with_item])
suite = Call(func, arglist)
suite.prefix = indent + (4 * " ")
return Node(syms.with_stmt,
[Name('with'),
with_item,
Name(':'),
Newline(),
suite])
def transform(self, func_node, results):
#new = func_node.clone()
new = func_node
_debug('transform(): Started for function ' + lined(self.def_name))
indent = ""
if len(self.comments) == 0:
self.reset_state()
return new
for child in new.children:
if child.type == self.syms.suite:
suite_node = child
_debug("Found suite! Indent is |" + indent + "|")
found_ind = False
for cc2 in suite_node.children:
# Need to skip first NEWLINE token
if found_ind is False:
if cc2.type == token.INDENT:
indent = cc2.value # Preserve correct indent
_debug("In suite.children! Updated indent to |" +
indent + "|")
found_ind = True
else:
func_name = self.get_func_name(func_node)
comm_struct = self.comments[0]
curr_comments = comm_struct.comments
if func_name not in comm_struct.no_funcs:
if self.re_hash_space.search(curr_comments):
indented_text = curr_comments.replace(
'# ', indent)
else:
indented_text = curr_comments.replace(
'\n ', '\n' + indent)
indented_text = self.format_docstring(
func_name, indented_text, indent)
comments = (indent + '""" ' + indented_text +
'\n' + indent + '"""')
suite_children = [String(comments), Newline()]
suite_node.insert_child(
1, pytree.Node(syms.simple_stmt,
suite_children))
self.comments.pop(0)
else:
# If comment was found after current func, we want
# to keep it for the next function, else restore
if comm_struct.func_name != func_name:
self.restore_last_comment()
self.reset_state()
return func_node
break
_debug("Breaking outer for-loop after suite")
break
_debug("New node would be |" + str(new) + "|")
self.reset_state()
return new
def finish_tree(self, tree, filename):
if not self._names:
return
names = [Leaf(token.LBRACE, "[", prefix=" "), Newline()]
for name in self._names:
names.append(String('"' + name + '"', prefix=" "))
names.append(Comma())
names.append(Newline())
names.append(Leaf(token.LBRACE, "]", prefix=""))
tree.append_child(Assign(Name("__all__"), names))
tree.append_child(Newline())
super(FixAllAttribute, self).finish_tree(tree, filename)
def touch_import_top(package, name_to_import, node):
"""Works like `does_tree_import` but adds an import statement at the
top if it was not imported (but below any __future__ imports).
Calling this multiple times adds them in reverse order.
Based on lib2to3.fixer_util.touch_import()
"""
root = find_root(node)
if does_tree_import(package, name_to_import, root):
return
# Look for __future__ imports and insert below them
found = False
for name in [
'absolute_import', 'division', 'print_function', 'unicode_literals'
]:
if does_tree_import('__future__', name, root):
found = True
break
if found:
# At least one __future__ import. We want to loop until we've seen them
# all.
start, end = None, None
for idx, node in enumerate(root.children):
if check_future_import(node):
start = idx
# Start looping
idx2 = start
while node:
node = node.next_sibling
idx2 += 1
if not check_future_import(node):
end = idx2
break
break
assert start is not None
assert end is not None
insert_pos = end
else:
# No __future__ imports
for idx, node in enumerate(root.children):
if node.type == syms.simple_stmt: # and node.children and node.children[0].type == token.STRING):
break
insert_pos = idx
if package is None:
import_ = Node(syms.import_name, [
Leaf(token.NAME, u"import"),
Leaf(token.NAME, name_to_import, prefix=u" ")
])
else:
import_ = FromImport(package,
[Leaf(token.NAME, name_to_import, prefix=u" ")])
children = [import_, Newline()]
root.insert_child(insert_pos, Node(syms.simple_stmt, children))
def add_future_import(tree, name):
"""Add future import.
From: https://github.com/facebook/tornado
Copyright 2009 Facebook
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
if not isinstance(tree, Node):
# Empty files (usually __init__.py) show up as a single Leaf
# instead of a Node, so leave them alone
return
first_stmt = tree.children[0]
if is_docstring(first_stmt):
# Skip a line and add the import after the docstring
tree.insert_child(1, Newline())
pos = 2
elif first_stmt.prefix:
# No docstring, but an initial comment (perhaps a #! line).
# Transfer the initial comment to a new blank line.
newline = Newline()
newline.prefix = first_stmt.prefix
first_stmt.prefix = ''
tree.insert_child(0, newline)
pos = 1
else:
# No comments or docstring, just insert at the start
pos = 0
tree.insert_child(
pos,
FromImport('__future__', [Name(name, prefix=' ')]))
tree.insert_child(pos + 1, Newline()) # terminates the import stmt
def add_globals(self, node):
"""Add required globals to the root of node. Idempotent."""
if self.added_pyi_globals:
return
# TODO: get rid of this -- added to prevent adding .parsed_pyi.top_lines every time
# we annotate a different function in the same file, but can break when we run the tool
# twice on the same file. Have to do something like what touch_import does.
self.added_pyi_globals = True
imports, top_lines = self.parsed_pyi.imports, self.parsed_pyi.top_lines
# Copy imports if not already present
for pkg, names in imports:
if names is None:
# TODO: do ourselves, touch_import puts stuff above license headers
touch_import(None, pkg, node) # == 'import pkg'
else:
for name in names:
touch_import(pkg, name, node)
root = find_root(node)
import_idx = [
idx for idx, node in enumerate(root.children)
if self.import_pattern.match(node)
]
if import_idx:
future_insert_pos = import_idx[0]
top_insert_pos = import_idx[-1] + 1
else:
future_insert_pos = top_insert_pos = 0
# first string (normally docstring)
for idx, node in enumerate(root.children):
if (node.type == syms.simple_stmt and node.children
and node.children[0].type == token.STRING):
future_insert_pos = top_insert_pos = idx + 1
break
top_lines = '\n'.join(top_lines)
top_lines = Util.parse_string(top_lines) # strips some newlines
for offset, node in enumerate(top_lines.children[:-1]):
root.insert_child(top_insert_pos + offset, node)
# touch_import doesn't do proper order for __future__
pkg = '__future__'
future_imports = [
n for n in self.future_imports
if not does_tree_import(pkg, n, root)
]
for offset, name in enumerate(future_imports):
node = FromImport(pkg, [Leaf(token.NAME, name, prefix=" ")])
node = Node(syms.simple_stmt, [node, Newline()])
root.insert_child(future_insert_pos + offset, node)
def transform(self, node, results):
u"""
a,b,c,d,e,f,*g,h,i = range(100) changes to
_3to2list = list(range(100))
a,b,c,d,e,f,g,h,i, = _3to2list[:6] + [_3to2list[6:-2]] + _3to2list[-2:]
and
for a,b,*c,d,e in iter_of_iters: do_stuff changes to
for _3to2iter in iter_of_iters:
_3to2list = list(_3to2iter)
a,b,c,d,e, = _3to2list[:2] + [_3to2list[2:-2]] + _3to2list[-2:]
do_stuff
"""
self.LISTNAME = self.new_name(u"_3to2list")
self.ITERNAME = self.new_name(u"_3to2iter")
expl, impl = results.get_products(u"expl"), results.get_products(
u"impl")
if expl is not None:
setup_line, power_line = self.fix_explicit_context(node, results)
setup_line.prefix = expl.prefix
power_line.prefix = indentation(expl.parent)
setup_line.append_child(Newline())
parent = node.parent
i = node.remove()
parent.insert_child(i, power_line)
parent.insert_child(i, setup_line)
elif impl is not None:
setup_line, power_line = self.fix_implicit_context(node, results)
suitify(node)
suite = [k for k in node.children if k.type == syms.suite][0]
setup_line.prefix = u""
power_line.prefix = suite.children[1].value
suite.children[2].prefix = indentation(suite.children[2])
suite.insert_child(2, Newline())
suite.insert_child(2, power_line)
suite.insert_child(2, Newline())
suite.insert_child(2, setup_line)
results.get_products(u"lst").replace(
Name(self.ITERNAME, prefix=u" "))
def match(self, node):
if (node.type in (token.COLON, token.COMMA, token.SEMI)
and node.get_suffix() != " "):
# If there is a newline after, no space
if (node.get_suffix().find('\n') == 0
or (node.next_sibling and node.next_sibling.children
and node.next_sibling.children[0] == Newline())):
return False
# If we are using slice notation, no space necessary
if node.parent.type in [symbols.subscript, symbols.sliceop]:
return False
return True
return False
def create_type_checking_import(package, name, node):
# type: (str, str, Node) -> None
"""
Create import statement of the form `from <package> import <name>` within a TYPING_CHECK
block
Parameters
-------------
package : str
name : str
Name of type being imported
node : Node
"""
def is_type_checking_decl(node):
# [Grammar]
# if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
if not node.type == syms.if_stmt:
return False
stmt = str(node.children[1]).strip()
if stmt in ('typing.TYPE_CHECKING', 'TYPE_CHECKING'):
return True
return False
root = find_root(node)
# figure out where to insert the new import. First try to find
# the first import and then skip to the last one.
type_checking_suite = None
parent = root
for idx, node in enumerate(root.children):
if not is_type_checking_decl(node):
continue
type_checking_suite = node.children[3]
parent = type_checking_suite
insert_pos = len(type_checking_suite.children) - 1
break
if type_checking_suite is None:
# Generate a new TYPE_CHECKING block at the bottom of the current import block and return
insert_pos = _get_bottom_of_imports(root)
_new_type_check_with_import(package, name, root, insert_pos)
return
import_ = _generate_import_node(package, name, prefix=" ")
children = [import_, Newline()]
parent.insert_child(insert_pos, Node(syms.simple_stmt, children))
def RaisesRegexOp(context, designator, exceptionClass, expected_regex,
indent, kws, arglist):
arglist = [a.clone() for a in arglist.children]
del arglist[2:4] # remove pattern and comma
arglist = Node(syms.arglist, arglist)
with_stmt = RaisesOp(context, exceptionClass, indent, kws, arglist)
with_stmt.insert_child(2, Name('as', prefix=" "))
with_stmt.insert_child(3, Name(designator, prefix=" "))
return Node(syms.suite,
[with_stmt,
Newline(),
Name('assert re.search(pattern, %s.value)' % designator,
prefix=indent)
])
def visit_NEWLINE(self, node):
if node.prefix.lstrip().startswith(self.marker):
# MEMO: <expr> -> _ = <expr>
target = node
while True:
parent = target.parent
if parent is None:
return
if type_repr(target.parent.type) == "simple_stmt":
break
target = parent
eol = target # target is Leaf("\n]")
target = eol.prev_sibling
cloned = target.clone()
cloned.parent = None
assigned = Assign(Name("_"), cloned)
assigned.prefix = target.prefix
target.replace(assigned)
# MEMO: adding print(SEP_MARKER, _, SEP_MARKER, sep="\n")
this_stmt = eol.parent
print_stmt = this_stmt.clone()
print_stmt.children = []
print_stmt.append_child(
Name(
"print({ms!r}, repr(_), {me!r}, sep='')".format(
ms="{}{}:".format(SEP_MARKER, node.get_lineno()), me=SEP_MARKER
)
)
)
print_stmt.prefix = assigned.prefix
# xxx: for first line
if not print_stmt.prefix:
prev_line = assigned.parent.prev_sibling
if prev_line is not None and prev_line.type == token.INDENT:
print_stmt.prefix = prev_line.value
print_stmt.append_child(Newline())
for i, stmt in enumerate(this_stmt.parent.children):
if stmt == this_stmt:
this_stmt.parent.insert_child(i + 1, print_stmt)
break
self.prev_newline = node
def add_import(import_name, node):
suite = get_parent_of_type(node, syms.suite)
test_case = suite
while test_case.parent.type != syms.file_input:
test_case = test_case.parent
file_input = test_case.parent
if not does_tree_import(None, import_name, node):
import_stmt = Node(syms.simple_stmt, [
Node(
syms.import_name,
[Name('import'), Name(import_name, prefix=' ')]),
Newline(),
])
insert_import(import_stmt, test_case, file_input)
def __insertImport(self, module, offset):
if self.__coreImport is None:
return
importLine = Node(syms.simple_stmt, [
Node(syms.import_name, [
Leaf(token.NAME, u"import"),
Leaf(token.NAME, module, prefix=" ")
]),
Newline()
])
self.__coreImport.parent.insert_child(
self.__coreImport.parent.children.index(self.__coreImport) +
offset, importLine)
def RaisesOp(context, exceptionClass, indent, kws, arglist, node):
exceptionClass.prefix = ""
args = [exceptionClass]
# Add match keyword arg to with statement if an expected regex was provided.
# In py27 the keyword is `expected_regexp`, in py3 is `expected_regex`
if 'expected_regex' in kws or 'expected_regexp' in kws:
expected_regex = kws.get('expected_regex',
kws.get('expected_regexp')).clone()
expected_regex.prefix = ''
args.append(String(', '))
args.append(KeywordArg(Name('match'), expected_regex))
with_item = Call(Name(context), args)
with_item.prefix = " "
args = []
arglist = [a.clone() for a in arglist.children[4:]]
if arglist:
arglist[0].prefix = ""
func = None
# :fixme: this uses hardcoded parameter names, which may change
if 'callableObj' in kws:
func = kws['callableObj']
elif 'callable_obj' in kws:
func = kws['callable_obj']
elif kws['args']: # any arguments assigned to `*args`
func = kws['args'][0]
else:
func = None
if func is None:
# Context manager
return Node(syms.with_stmt, [with_item])
if func.type == syms.lambdef:
suite = func.children[-1].clone()
else:
# TODO: Newlines within arguments are not handled yet.
# If argment prefix contains a newline, all whitespace around this
# ought to be replaced by indent plus 4+1+len(func) spaces.
suite = Call(func, arglist)
suite.prefix = indent + (4 * " ")
return Node(
syms.with_stmt,
[Name('with'), with_item,
Name(':'), Newline(), suite])
def Def(name, args, *body, prefix=""):
return Node(syms.funcdef, [
Name("def", prefix=prefix),
maybe_name(name),
Node(syms.parameters, [
LParen(),
args,
RParen(),
]),
Colon(),
Node(syms.suite, [
Newline(),
Indent(),
*body,
Dedent(),
])
])
def fix_submod_import(self, imported, name, node):
u"""
Accepts a list of NAME leafs, a name string, and a node
node is given as an argument to BaseFix.transform()
NAME leafs come from an import_as_names node (the children)
name string is the base name found in node.
"""
submods = []
missed = []
for attr in imported:
dotted = u'.'.join((name, attr.value))
if dotted in MAPPING:
# get the replacement module
to_repl = MAPPING[dotted]
if u'.' not in to_repl:
# it's a simple name, so use a simple replacement.
_import = NameImport(Name(to_repl, prefix=u" "),
attr.value)
submods.append(_import)
elif attr.type == token.NAME:
missed.append(attr.clone())
if not submods:
return
parent = node.parent
node.replace(submods[0])
if len(submods) > 1:
start = submods.pop(0)
prev = start
for submod in submods:
parent.append_child(submod)
if missed:
self.warning(
node,
u"Imported names not known to 3to2 to be part of the package %s. Leaving those alone... high probability that this code will be incorrect."
% (name))
children = [
Name(u"from"),
Name(name, prefix=u" "),
Name(u"import", prefix=u" "),
Node(syms.import_as_names, missed)
]
orig_stripped = Node(syms.import_from, children)
parent.append_child(Newline())
parent.append_child(orig_stripped)
def add_future(node, symbol):
root = find_root(node)
for idx, node in enumerate(root.children):
if node.type == syms.simple_stmt and \
len(node.children) > 0 and node.children[0].type == token.STRING:
# skip over docstring
continue
names = check_future_import(node)
if not names:
# not a future statement; need to insert before this
break
if symbol in names:
# already imported
return
import_ = FromImport('__future__', [Leaf(token.NAME, symbol, prefix=" ")])
children = [import_, Newline()]
root.insert_child(idx, Node(syms.simple_stmt, children))