def parse(code):
"""String -> AST
Parse the string and return its AST representation. May raise
a ParseError exception.
"""
added_newline = False
if not code.endswith("\n"):
code += "\n"
added_newline = True
try:
drv = driver.Driver(pygram.python_grammar, pytree.convert)
result = drv.parse_string(code, True)
except ParseError:
log.debug("Had problems parsing:\n%s\n" % quoted_block(code))
raise
# Always return a Node, not a Leaf.
if isinstance(result, Leaf):
result = Node(syms.file_input, [result])
result.added_newline = added_newline
return result
def fix_not_a_in_b(node: LN, capture: Capture, filename: Filename):
capture["element"].parent = None
capture["collection"].parent = None
new_comparison = Node(
comparison,
[
capture["element"],
Node(comp_op,
[Leaf(1, "not", prefix=" "),
Leaf(1, "in", prefix=" ")]),
capture["collection"],
],
)
new_comparison.parent = node.parent
return new_comparison
def get_annotated_param(node, arg, *, missing_ok=False):
if node.type not in (token.NAME, syms.tname):
raise NotImplementedError(f"unexpected node token: `{node}`")
actual_ann = None
if node.type == syms.tname:
actual_ann = node.children[2]
node = node.children[0]
if not isinstance(node, Leaf) or arg.arg != node.value:
raise ValueError(
f".pyi file expects argument {arg.arg!r} next but argument " +
f"{minimize_whitespace(str(node))!r} found in source")
if arg.annotation is None:
if actual_ann is None:
if missing_ok:
return new(node)
raise ValueError(
f".pyi file is missing annotation for {arg.arg!r} and source "
+ f"doesn't provide it either")
ann = new(actual_ann)
else:
ann = convert_annotation(arg.annotation)
ann.prefix = ' '
if actual_ann is not None and actual_ann != ann:
ann_str = minimize_whitespace(str(ann))
actual_ann_str = minimize_whitespace(str(actual_ann))
raise ValueError(
f"incompatible annotation for {arg.arg!r}. Expected: " +
f"{ann_str!r}, actual: {actual_ann_str!r}")
return Node(syms.tname, [new(node), new(_colon), ann])
def add_future(node, symbol):
root = fixer_util.find_root(node)
for idx, node in enumerate(root.children):
if node.type == syms.simple_stmt and \
len(node.children) > 0 and node.children[0].type == token.STRING:
# skip over docstring
continue
names = check_future_import(node)
if not names:
# not a future statement; need to insert before this
break
if symbol in names:
# already imported
return
import_ = fixer_util.FromImport('__future__',
[Leaf(token.NAME, symbol, prefix=" ")])
# Place after any comments or whitespace. (copyright, shebang etc.)
import_.prefix = node.prefix
node.prefix = ''
children = [import_, fixer_util.Newline()]
root.insert_child(idx, Node(syms.simple_stmt, children))
def append_credentials(src,
access_key,
secret_key,
target='CREDENTIALS',
comment='Added by cornac'):
if isinstance(src, str):
if not src.endswith('\n'):
src += '\n'
src = parse_py(src)
for varname, atom in iter_assign(src):
if varname != target:
continue
if len(atom.children) == 2:
# Atom is an empty dict.
dictmaker = Node(syms.dictsetmaker, [])
atom.children.insert(1, dictmaker)
if atom.children[1].type != syms.dictsetmaker:
raise ValueError(f"{target} is not a plain dict.")
append_to_dict(atom, key=access_key, value=secret_key, comment=comment)
break
else:
raise ValueError(f"{target} is not set.")
return src
def _c_tuple(tup):
contents = [_convert_annotation(elt) for elt in tup.elts]
for index in range(len(contents) - 1, 0, -1):
contents[index].prefix = " "
contents.insert(index, new(_comma))
return Node(syms.subscriptlist, contents)
def future_import2(feature, node):
"""
An alternative to future_import() which might not work ...
"""
root = find_root(node)
if does_tree_import(u"__future__", feature, node):
return
insert_pos = 0
for idx, node in enumerate(root.children):
if node.type == syms.simple_stmt and node.children and \
node.children[0].type == token.STRING:
insert_pos = idx + 1
break
for thing_after in root.children[insert_pos:]:
if thing_after.type == token.NEWLINE:
insert_pos += 1
continue
prefix = thing_after.prefix
thing_after.prefix = u""
break
else:
prefix = u""
import_ = FromImport(u"__future__",
[Leaf(token.NAME, feature, prefix=u" ")])
children = [import_, Newline()]
root.insert_child(insert_pos,
Node(syms.simple_stmt, children, prefix=prefix))
def add_global_assignment_after_imports(_name, assignment, node):
"""
Big copy paste + modification from touch_import
"""
root = find_root(node)
if find_binding(_name, root):
return
# figure out where to insert the assignment.
# First try to find the first import and then skip to the last one.
insert_pos = offset = 0
for idx, node in enumerate(root.children):
if not is_import_ish_stmt(node):
continue
for offset, node2 in enumerate(root.children[idx:]):
if not is_import_ish_stmt(node2):
break
insert_pos = idx + offset
break
# if there are no imports where we can insert, find the docstring.
# if that also fails, we stick to the beginning of the file
if insert_pos == 0:
for idx, node in enumerate(root.children):
if (node.type == syms.simple_stmt and node.children
and node.children[0].type == token.STRING):
insert_pos = idx + 1
break
children = [assignment, Newline()]
root.insert_child(insert_pos, Node(syms.simple_stmt, children))
def transform(self, node, results):
args = []
item_id = 0
while item_id < len(node.children):
if (node.children[item_id].__class__.__name__ == "Leaf"):
if node.children[item_id].value == ':':
#加花括号
args += [
Leaf(token.LBRACE, '{'),
node.children[item_id + 1].clone(),
Leaf(token.RBRACE, '}'),
Leaf(4, '\r\n')
]
item_id += 1
#if，elif处理
elif (node.children[item_id].value
== 'elif') | (node.children[item_id].value == 'if'):
if node.children[item_id].value == 'elif':
leaf_temp = Leaf(1, 'if')
leaf_temp.prefix = " "
args += [Leaf(1, 'else'), leaf_temp]
else:
args += [node.children[item_id].clone()]
args += [Leaf(7, '(')]
args += [node.children[item_id + 1].clone()]
args += [Leaf(8, ')')]
item_id += 1
else:
args += [node.children[item_id].clone()]
else:
print("\nFixerError_if\n")
item_id += 1
new = Node(node.type, args)
new.prefix = node.prefix
return new
def __insertImport(self, module, offset):
if self.__coreImport is None:
return
importLine = Node(syms.simple_stmt, [
Node(syms.import_name, [
Leaf(token.NAME, u"import"),
Leaf(token.NAME, module, prefix=" ")
]),
Newline()
])
self.__coreImport.parent.insert_child(
self.__coreImport.parent.children.index(self.__coreImport) +
offset, importLine)
def transform(self, node, results):
u"""
Call __builtins__.long() with the value and the base of the value.
This works because 0b10 is int("10", 2), 0o10 is int("10", 8), etc.
"""
val = node.value
base_ = base(val)
if base_ == 8:
assert val.strip().startswith(u"0o") or \
val.strip().startswith(
u"0O"), u"Invalid format for octal literal"
node.changed()
node.value = u"".join((u"0", val[2:]))
elif base_ == 2:
assert val.startswith(u"0") and val[1] in u"bB", \
u"Invalid format for binary literal"
# __builtins__.long
func_name = Node(syms.power,
Attr(Name(u"__builtins__"), Name(u"long")))
# ("...", 2)
func_args = [
String(u"".join((u"\"", val.strip()[2:], u"\""))),
Comma(),
Number(2, prefix=u" ")
]
new_node = Call(func_name, func_args, node.prefix)
return new_node
def suitify(parent):
u"""
Turn the stuff after the first colon in parent's children
into a suite, if it wasn't already
"""
for node in parent.children:
if node.type == syms.suite:
# already in the prefered format, do nothing
return
# One-liners have no suite node, we have to fake one up
for i, node in enumerate(parent.children):
if node.type == token.COLON:
break
else:
raise ValueError(u"No class suite and no ':'!")
# Move everything into a suite node
suite = Node(syms.suite, [
Newline(),
Leaf(token.INDENT,
indentation(node) + indentation_step(node))
])
one_node = parent.children[i + 1]
one_node.remove()
one_node.prefix = u''
suite.append_child(one_node)
parent.append_child(suite)
def transform(self, node, results):
unifunc = results["unifunc"]
strfunc = Name("__str__", prefix=unifunc.prefix)
unifunc.replace(strfunc)
klass = node.clone()
klass.prefix = '\n' + find_indentation(node)
decorator = Node(
syms.decorator,
[Leaf(token.AT, "@"),
Name('python_2_unicode_compatible')])
decorated = Node(syms.decorated, [decorator, klass],
prefix=node.prefix)
node.replace(decorated)
touch_import('django.utils.encoding', 'python_2_unicode_compatible',
decorated)
def ImportAsName(name, as_name, prefix=None):
new_name = Name(name)
new_as = Name(u"as", prefix=u" ")
new_as_name = Name(as_name, prefix=u" ")
new_node = Node(syms.import_as_name, [new_name, new_as, new_as_name])
if prefix is not None:
new_node.prefix = prefix
return new_node
def get_dotted_import_replacement(self, name_node, attr_node, mapping=MAPPING, renamed=None):
u"""
For (http, client) given and httplib being the correct replacement,
returns (httplib as client, None)
For (test, support) given and test.test_support being the replacement,
returns (test, test_support as support)
"""
full_name = name_node.value + u'.' + attr_node.value
replacement = mapping[full_name]
if u'.' in replacement:
new_name, new_attr = replacement.split(u'.')
if renamed is None:
return Name(new_name, prefix=name_node.prefix), Node(syms.dotted_as_name, [Name(new_attr, prefix=attr_node.prefix), Name(u'as', prefix=u" "), attr_node.clone()])
else:
return Name(new_name, prefix=name_node.prefix), Name(new_attr, prefix=attr_node.prefix)
else:
return Node(syms.dotted_as_name, [Name(replacement, prefix=name_node.prefix), Name(u'as', prefix=u' '), Name(attr_node.value, prefix=attr_node.prefix)]), None
def Def(name, args, *body, prefix=""):
return Node(syms.funcdef, [
Name("def", prefix=prefix),
maybe_name(name),
Node(syms.parameters, [
LParen(),
args,
RParen(),
]),
Colon(),
Node(syms.suite, [
Newline(),
Indent(),
*body,
Dedent(),
])
])
def _Parse(cls, code):
"""
Parses the given code string returning its lib2to3 AST tree.
:return lib2to3.AST:
Returns the lib2to3 AST.
"""
def _GetLastLeaf(node):
from lib2to3.pytree import Leaf
r_leaf = node
while not isinstance(r_leaf, Leaf):
r_leaf = r_leaf.children[-1]
return r_leaf
# Prioritary import.
# Other imports
from zerotk.reraiseit import reraise
from lib2to3 import pygram, pytree
from lib2to3.pgen2 import driver
from lib2to3.pgen2.parse import ParseError
from lib2to3.pygram import python_symbols
from lib2to3.pytree import Leaf, Node
from lib2to3.refactor import _detect_future_features
added_newline = code and not code.endswith('\n')
if added_newline:
code += '\n'
# Selects the appropriate grammar depending on the usage of
# "print_function" future feature.
future_features = _detect_future_features(code)
if 'print_function' in future_features:
grammar = pygram.python_grammar_no_print_statement
else:
grammar = pygram.python_grammar
try:
drv = driver.Driver(grammar, pytree.convert)
result = drv.parse_string(code, True)
except ParseError as e:
reraise(e, "Had problems parsing:\n%s\n" % cls._QuotedBlock(code))
# Always return a Node, not a Leaf.
if isinstance(result, Leaf):
result = Node(python_symbols.file_input, [result])
# Remove AST-leaf for the added newline.
if added_newline:
last_leaf = _GetLastLeaf(result)
if not (last_leaf.type == 0 and last_leaf.value == ''):
if last_leaf.prefix:
last_leaf.prefix = last_leaf.prefix[:-1]
else:
last_leaf.remove()
return result
def fix_submod_import(self, imported, name, node):
u"""
Accepts a list of NAME leafs, a name string, and a node
node is given as an argument to BaseFix.transform()
NAME leafs come from an import_as_names node (the children)
name string is the base name found in node.
"""
submods = []
missed = []
for attr in imported:
dotted = u'.'.join((name, attr.value))
if dotted in MAPPING:
# get the replacement module
to_repl = MAPPING[dotted]
if u'.' not in to_repl:
# it's a simple name, so use a simple replacement.
_import = NameImport(Name(to_repl, prefix=u" "),
attr.value)
submods.append(_import)
elif attr.type == token.NAME:
missed.append(attr.clone())
if not submods:
return
parent = node.parent
node.replace(submods[0])
if len(submods) > 1:
start = submods.pop(0)
prev = start
for submod in submods:
parent.append_child(submod)
if missed:
self.warning(
node,
u"Imported names not known to 3to2 to be part of the package %s. Leaving those alone... high probability that this code will be incorrect."
% (name))
children = [
Name(u"from"),
Name(name, prefix=u" "),
Name(u"import", prefix=u" "),
Node(syms.import_as_names, missed)
]
orig_stripped = Node(syms.import_from, children)
parent.append_child(Newline())
parent.append_child(orig_stripped)
def transform(self, node, results):
args = [Leaf(1, 'while')]
args += [Leaf(7, '(')]
args += [n.clone() for n in results["test_content"]]
args += [Leaf(8, ')'),Leaf(token.LBRACE, '{')]
args += [n.clone() for n in results["content"]]
new = Node(node.type,args)
new.prefix = node.prefix
return new
def _c_keyword(kwarg):
assert kwarg.arg
return Node(
syms.argument,
[
Leaf(token.NAME, kwarg.arg),
new(_eq, prefix=''),
convert_annotation(kwarg.value),
],
)
def add_end_part(end, file, parent, loc):
if isNone(end):
return
if end.type == token.STRING and end.value in ("' '", '" "', "u' '", 'u" "',
"b' '", 'b" "'):
return
if file is None:
touch_import(None, "sys", parent)
file = Node(syms.power,
[Name("sys"),
Node(syms.trailer, [Dot(), Name("stdout")])])
end_part = Node(syms.power, [
file,
Node(syms.trailer, [Dot(), Name("write")]),
Node(syms.trailer, [LParen(), end, RParen()])
])
end_part.prefix = " "
parent.insert_child(loc, Leaf(token.SEMI, ";"))
parent.insert_child(loc + 1, end_part)
def make_import(*names, from_module=None):
assert names
imports = []
if from_module:
statement = syms.import_from
container = syms.import_as_names
single = syms.import_as_name
result = [
Leaf(token.NAME, 'from'),
Leaf(token.NAME, from_module, prefix=' '),
Leaf(token.NAME, 'import', prefix=' '),
]
else:
statement = syms.import_name
container = syms.dotted_as_names
single = syms.dotted_as_name
result = [Leaf(token.NAME, 'import')]
for alias in names:
name = Leaf(token.NAME, alias.name, prefix=' ')
if alias.asname:
_as = Leaf(token.NAME, 'as', prefix=' ')
asname = Leaf(token.NAME, alias.asname, prefix=' ')
imports.append(Node(single, [name, _as, asname]))
else:
imports.append(name)
if len(imports) == 1:
result.append(imports[0])
else:
imports_and_commas = []
for imp in imports[:-1]:
imports_and_commas.append(imp)
imports_and_commas.append(Leaf(token.COMMA, ','))
imports_and_commas.append(imports[-1])
result.append(Node(container, imports_and_commas))
return Node(
syms.simple_stmt,
[
Node(statement, result),
Leaf(token.NEWLINE, '\n'), # FIXME: \r\n?
],
)
def transform(self, node, results):
args = []
as_list = [[], []]
for i in node.children:
if i.type == 11:
args.append(Leaf(token.LBRACE, '{'))
elif i.type == syms.suite:
#as 语句
for j in range(len(as_list[0])):
args.append(i.children[0].clone())
args.append(i.children[1].clone())
args.append(Node(syms.simple_stmt,[\
Leaf(1, 'auto'),
as_list[0][j].clone(),
Leaf(22, '='),
as_list[1][j].clone(),
Leaf(1, ';'),]))
as_list[0] = []
as_list[1] = []
args.append(i.clone())
args.append(Leaf(token.RBRACE, '}'))
args.append(Leaf(4, '\r\n'))
args.append(i.children[-1].clone())
elif i.type == syms.except_clause:
except_args = [Leaf(1, 'catch'),\
Leaf(7, '(')]
for j in i.children[1:]:
#as 语句
if (j.type == 1) & (j.value == 'as'):
as_list[0].append(except_args[-1].clone())
elif len(as_list[0]) != len(as_list[1]):
as_list[1].append(j.clone())
else:
except_args.append(j.clone())
except_args.append(Leaf(8, ')'))
args.append(
Node(syms.except_clause, except_args, prefix=i.prefix))
else:
args.append(i.clone())
new = Node(node.type, args)
new.prefix = node.prefix
return new
def add_globals(self, node):
"""Add required globals to the root of node. Idempotent."""
if self.added_pyi_globals:
return
# TODO: get rid of this -- added to prevent adding .parsed_pyi.top_lines every time
# we annotate a different function in the same file, but can break when we run the tool
# twice on the same file. Have to do something like what touch_import does.
self.added_pyi_globals = True
imports, top_lines = self.parsed_pyi.imports, self.parsed_pyi.top_lines
# Copy imports if not already present
for pkg, names in imports:
if names is None:
# TODO: do ourselves, touch_import puts stuff above license headers
touch_import(None, pkg, node) # == 'import pkg'
else:
for name in names:
touch_import(pkg, name, node)
root = find_root(node)
import_idx = [
idx for idx, node in enumerate(root.children)
if self.import_pattern.match(node)
]
if import_idx:
future_insert_pos = import_idx[0]
top_insert_pos = import_idx[-1] + 1
else:
future_insert_pos = top_insert_pos = 0
# first string (normally docstring)
for idx, node in enumerate(root.children):
if (node.type == syms.simple_stmt and node.children
and node.children[0].type == token.STRING):
future_insert_pos = top_insert_pos = idx + 1
break
top_lines = '\n'.join(top_lines)
top_lines = Util.parse_string(top_lines) # strips some newlines
for offset, node in enumerate(top_lines.children[:-1]):
root.insert_child(top_insert_pos + offset, node)
# touch_import doesn't do proper order for __future__
pkg = '__future__'
future_imports = [
n for n in self.future_imports
if not does_tree_import(pkg, n, root)
]
for offset, name in enumerate(future_imports):
node = FromImport(pkg, [Leaf(token.NAME, name, prefix=" ")])
node = Node(syms.simple_stmt, [node, Newline()])
root.insert_child(future_insert_pos + offset, node)
def NameImport(package, as_name=None, prefix=None):
"""
Accepts a package (Name node), name to import it as (string), and
optional prefix and returns a node:
import <package> [as <as_name>]
"""
if prefix is None:
prefix = u""
children = [Name(u"import", prefix=prefix), package]
if as_name is not None:
children.extend([Name(u"as", prefix=u" "), Name(as_name, prefix=u" ")])
return Node(syms.import_name, children)
def parse_string(cls, text):
"""Use lib2to3 to parse text into a Node."""
text = text.strip()
if not text:
# cls.driver.parse_string just returns the ENDMARKER Leaf, wrap in
# a Node for consistency
return Node(syms.file_input, [Leaf(token.ENDMARKER, '')])
# workaround: parsing text without trailing '\n' throws exception
text += '\n'
return cls.driver.parse_string(text)
def transform(self, node, results):
args = [Leaf(1, 'for')]
args += [Leaf(7, '(')]
args += [Leaf(1, 'auto')] #c++11
args += [n.clone() for n in results["in_before"]]
args += [Leaf(token.COLON, ':')]
args += [n.clone() for n in results["in_after"]]
args += [Leaf(8, ')'), Leaf(token.LBRACE, '{')]
args += [n.clone() for n in results["content"]]
new = Node(node.type, args)
new.prefix = node.prefix
return new
def _c_call(call):
contents = [convert_annotation(arg) for arg in call.args]
contents.extend(convert_annotation(kwarg) for kwarg in call.keywords)
for index in range(len(contents) - 1, 0, -1):
contents[index].prefix = " "
contents.insert(index, new(_comma))
call_args = [
new(_lpar),
new(_rpar),
]
if contents:
call_args.insert(1, Node(syms.arglist, contents))
return Node(
syms.power,
[convert_annotation(call.func),
Node(
syms.trailer,
call_args,
)],
)
def create_import(import_desc):
"""Create an AST representing import statement from given description.
>>> regenerate(create_import("unittest"))
'import unittest\\n'
>>> regenerate(create_import(("nose", "SkipTest")))
'from nose import SkipTest\\n'
"""
if isinstance(import_desc, tuple):
package, name = import_desc
return Node(syms.import_from,
[Leaf(token.NAME, 'from'),
Leaf(token.NAME, package, prefix=" "),
Leaf(token.NAME, 'import', prefix=" "),
Leaf(token.NAME, name, prefix=" "),
Newline()])
else:
return Node(syms.import_name,
[Leaf(token.NAME, 'import'),
Leaf(token.NAME, import_desc, prefix=" "),
Newline()])
def transform_colon(self, node):
node_copy = node.clone()
# Strip any whitespace that could have been there
node_copy.prefix = node_copy.prefix.lstrip()
old_depth = find_indentation(node)
new_indent = '%s%s' % ((' ' * 4), old_depth)
new_node = Node(symbols.suite, [Leaf(token.NEWLINE, '\n'), Leaf(token
.INDENT, new_indent), node_copy, Leaf(token.DEDENT, '')])
node.replace(new_node)
node.changed()
# Replace node with new_node in case semi
return node_copy
