def test_remove(self):
l1 = pytree.Leaf(100, "foo")
l2 = pytree.Leaf(100, "foo")
n1 = pytree.Node(1000, [l1, l2])
n2 = pytree.Node(1000, [n1])
self.assertEqual(n1.remove(), 0)
self.assertEqual(n2.children, [])
self.assertEqual(l1.parent, n1)
self.assertEqual(n1.parent, None)
self.assertEqual(n2.parent, None)
self.assertFalse(n1.was_changed)
self.assertTrue(n2.was_changed)
self.assertEqual(l2.remove(), 1)
self.assertEqual(l1.remove(), 0)
self.assertEqual(n1.children, [])
self.assertEqual(l1.parent, None)
self.assertEqual(n1.parent, None)
self.assertEqual(n2.parent, None)
self.assertTrue(n1.was_changed)
self.assertTrue(n2.was_changed)
def add_version(tree) -> pytree.Leaf:
"""Add a version node"""
setup_args = _searching.find_setup_args(tree)
version_node = pytree.Node(
python_symbols.argument,
[
pytree.Leaf(token.NAME, "version"),
pytree.Leaf(22, "="),
pytree.Leaf(token.STRING, '""'),
],
)
add_arg(setup_args, version_node)
return version_node.children[-1]
def package_tree(self, package):
""" Return pytree tree for accessing the package
Example:
>>> package_tree("Orange.feature.scoring")
[Name('Orange'), trailer('.' 'feature'), trailer('.', 'scoring')]
"""
path = package.split('.')
nodes = [Name(path[0])]
for name in path[1:]:
new = pytree.Node(self.syms.trailer, [Dot(), Name(name)])
nodes.append(new)
return nodes
def transform(self, node, results):
head = results['head']
method = results['method'][0]
tail = results['tail']
syms = self.syms
method_name = method.value
if method_name == 'torsion':
method_name = 'dihedral'
head = [n.clone() for n in head]
tail = [n.clone() for n in tail]
args = head + [
pytree.Node(syms.trailer, [
Dot(),
Name(method_name, prefix=method.prefix),
Dot(),
Name('value'),
LParen(),
RParen()
])
]
new = pytree.Node(syms.power, args)
return new
def test_replace(self):
l1 = pytree.Leaf(100, "foo")
l2 = pytree.Leaf(100, "+")
l3 = pytree.Leaf(100, "bar")
n1 = pytree.Node(1000, [l1, l2, l3])
self.assertEqual(n1.children, [l1, l2, l3])
self.assertIsInstance(n1.children, list)
self.assertFalse(n1.was_changed)
l2new = pytree.Leaf(100, "-")
l2.replace(l2new)
self.assertEqual(n1.children, [l1, l2new, l3])
self.assertIsInstance(n1.children, list)
self.assertTrue(n1.was_changed)
def transform(self, node, results):
syms = self.syms
exc = results["exc"].clone()
if exc.type == token.STRING:
msg = "Python 3 does not support string exceptions"
self.cannot_convert(node, msg)
return
# Python 2 supports
# raise ((((E1, E2), E3), E4), E5), V
# as a synonym for
# raise E1, V
# Since Python 3 will not support this, we recurse down any tuple
# literals, always taking the first element.
if is_tuple(exc):
while is_tuple(exc):
# exc.children[1:-1] is the unparenthesized tuple
# exc.children[1].children[0] is the first element of the tuple
exc = exc.children[1].children[0].clone()
exc.prefix = u" "
if "val" not in results:
# One-argument raise
new = pytree.Node(syms.raise_stmt, [Name(u"raise"), exc])
new.prefix = node.prefix
return new
val = results["val"].clone()
if is_tuple(val):
args = [c.clone() for c in val.children[1:-1]]
else:
val.prefix = u""
args = [val]
return pytree.Node(syms.raise_stmt,
[Name(u"raise"), Call(exc, args)],
prefix=node.prefix)
def _WrapEndMarker(tree):
"""Wrap a single ENDMARKER token in a "file_input" node.
Arguments:
tree: (pytree.Node) The root node of the parsed tree.
Returns:
The root node of the parsed tree. If the tree is a single ENDMARKER node,
then that node is wrapped in a "file_input" node. That will ensure we don't
skip comments attached to that node.
"""
if isinstance(tree, pytree.Leaf) and tree.type == token.ENDMARKER:
return pytree.Node(pygram.python_symbols.file_input, [tree])
return tree
def test_node_insert_child(self):
l1 = pytree.Leaf(100, "foo")
n1 = pytree.Node(1000, [l1])
l2 = pytree.Leaf(100, "bar")
n1.insert_child(0, l2)
self.assertEqual(l2.parent, n1)
self.assertEqual(n1.children, [l2, l1])
l3 = pytree.Leaf(100, "abc")
n1.insert_child(2, l3)
self.assertEqual(n1.children, [l2, l1, l3])
# I don't care what it raises, so long as it's an exception
self.assertRaises(Exception, n1.insert_child, 0, list)
def test_node_append_child(self):
n1 = pytree.Node(1000, [])
l1 = pytree.Leaf(100, "foo")
n1.append_child(l1)
self.assertEqual(l1.parent, n1)
self.assertEqual(n1.children, [l1])
l2 = pytree.Leaf(100, "bar")
n1.append_child(l2)
self.assertEqual(l2.parent, n1)
self.assertEqual(n1.children, [l1, l2])
# I don't care what it raises, so long as it's an exception
self.assertRaises(Exception, n1.append_child, list)
def transform(self, node, results):
# First, find a the sys import. We'll just hope it's global scope.
if "sys_import" in results:
if self.sys_import is None:
self.sys_import = results["sys_import"]
return
func = results["func"].clone()
func.prefix = u""
register = pytree.Node(syms.power,
Attr(Name(u"atexit"), Name(u"register")))
call = Call(register, [func], node.prefix)
node.replace(call)
if self.sys_import is None:
# That's interesting.
self.warning(
node, "Can't find sys import; Please add an atexit "
"import at the top of your file.")
return
# Now add an atexit import after the sys import.
names = self.sys_import.children[1]
if names.type == syms.dotted_as_names:
names.append_child(Comma())
names.append_child(Name(u"atexit", u" "))
else:
containing_stmt = self.sys_import.parent
position = containing_stmt.children.index(self.sys_import)
stmt_container = containing_stmt.parent
new_import = pytree.Node(
syms.import_name,
[Name(u"import"), Name(u"atexit", u" ")])
new = pytree.Node(syms.simple_stmt, [new_import])
containing_stmt.insert_child(position + 1, Newline())
containing_stmt.insert_child(position + 2, new)
def _CreateCommentsFromPrefix(comment_prefix,
comment_lineno,
comment_column,
standalone=False):
"""Create pytree nodes to represent the given comment prefix.
Args:
comment_prefix: (unicode) the text of the comment from the node's prefix.
comment_lineno: (int) the line number for the start of the comment.
comment_column: (int) the column for the start of the comment.
standalone: (bool) determines if the comment is standalone or not.
Returns:
The simple_stmt nodes if this is a standalone comment, otherwise a list of
new COMMENT leafs. The prefix may consist of multiple comment blocks,
separated by blank lines. Each block gets its own leaf.
"""
# The comment is stored in the prefix attribute, with no lineno of its
# own. So we only know at which line it ends. To find out at which line it
# starts, look at how many newlines the comment itself contains.
comments = []
lines = comment_prefix.split('\n')
index = 0
while index < len(lines):
comment_block = []
while index < len(lines) and lines[index].lstrip().startswith('#'):
comment_block.append(lines[index])
index += 1
if comment_block:
new_lineno = comment_lineno + index - 1
comment_block[0] = comment_block[0].lstrip()
comment_block[-1] = comment_block[-1].rstrip('\n')
comment_leaf = pytree.Leaf(
type=token.COMMENT,
value='\n'.join(comment_block),
context=('', (new_lineno, comment_column)))
comment_node = comment_leaf if not standalone else pytree.Node(
pygram.python_symbols.simple_stmt, [comment_leaf])
comments.append(comment_node)
while index < len(lines) and not lines[index].lstrip():
index += 1
return comments
def add_arg(trailer, argument) -> None:
"""Add an argument to a call"""
try:
target = _searching.find_first_of_type(
trailer, (python_symbols.argument, python_symbols.arglist)
)
except _searching.TypeNotFoundError:
trailer.insert_child(1, argument)
else:
if target.type == python_symbols.argument:
new = pytree.Node(python_symbols.arglist, [])
old = target.clone()
new.append_child(old)
target.replace(new)
target = new
target.append_child(pytree.Leaf(token.COMMA, ","))
target.append_child(argument)
def _InsertPseudoParentheses(node):
"""Insert pseudo parentheses so that dicts can be formatted correctly."""
comment_node = None
if isinstance(node, pytree.Node):
if node.children[-1].type == token.COMMENT:
comment_node = node.children[-1].clone()
node.children[-1].remove()
first = _GetFirstLeafNode(node)
last = _GetLastLeafNode(node)
lparen = pytree.Leaf(token.LPAR,
u'(',
context=('', (first.get_lineno(), first.column - 1)))
last_lineno = last.get_lineno()
if last.type == token.STRING and '\n' in last.value:
last_lineno += last.value.count('\n')
if last.type == token.STRING and '\n' in last.value:
last_column = len(last.value.split('\n')[-1]) + 1
else:
last_column = last.column + len(last.value) + 1
rparen = pytree.Leaf(token.RPAR,
u')',
context=('', (last_lineno, last_column)))
lparen.is_pseudo = True
rparen.is_pseudo = True
if isinstance(node, pytree.Node):
node.insert_child(0, lparen)
node.append_child(rparen)
if comment_node:
node.append_child(comment_node)
_AppendFirstLeafTokenSubtype(node,
format_token.Subtype.DICTIONARY_VALUE)
else:
clone = node.clone()
new_node = pytree.Node(syms.atom, [lparen, clone, rparen])
node.replace(new_node)
_AppendFirstLeafTokenSubtype(clone,
format_token.Subtype.DICTIONARY_VALUE)
def transform(self, node, results):
member = results.get("member")
head = results.get("head")
tail = results.get("tail")
module = head[0].value
if member and module in self._modules_to_change:
node = member[0]
head = head[0]
old_name = module + "." + node.value
if old_name not in self.mapping:
return
new_name = unicode(self.mapping[old_name])
if ":" in new_name:
# ':' is the delimiter used to separate module namespace
package = new_name.split(":", 1)[0]
new_name = new_name.replace(":", ".")
else:
package = new_name.rsplit(".", 1)[0]
syms = self.syms
if tail:
tail = [t.clone() for t in tail]
new = self.package_tree(new_name)
new = pytree.Node(syms.power, new + tail, prefix=head.prefix)
# Make sure the proper package is imported
# if ":" in new_name:
# package = new_name.split(":",1)[0]
# else:
# package = new_name.rsplit(".", 1)[0]
def orange_to_root(package):
return "Orange" if package.startswith("Orange.") else package
touch_import(None, orange_to_root(package), node)
return new
def append_entry_to_atom(atom, entry):
"""Append an entry to a list"""
try:
target = _searching.find_first_of_type(
atom, (token.STRING, python_symbols.listmaker)
)
except _searching.TypeNotFoundError:
atom.insert_child(-1, pytree.Leaf(token.NAME, entry))
else:
if target.type == token.STRING:
context = (target.prefix, (target.lineno, target.column))
new = pytree.Node(
python_symbols.listmaker,
[pytree.Leaf(token.NAME, target.value, context)],
)
target.replace(new)
target = new
if not _searching.trailing_comma(target):
target.append_child(pytree.Leaf(token.COMMA, ","))
target.append_child(pytree.Leaf(token.STRING, entry))
def Visit_import_from(self, node):
"""
このようにimportが出力されてほしい
from foo import (
x,
y,
z,
)
from foo import (
x,
)
from foo import ( # NOQA
x,
y,
)
このために 以下の設定を追加してこのAssignerを使う。
- dedent_closing_brackets=true
"""
for around_last in reversed(node.children):
if around_last.type != token.COMMENT:
break
# node.type < 256は literal的なnode
if around_last.type < 256 and around_last.value == ")":
as_names = node.children[-2]
if not as_names.children:
# `from foo import (x)` to `from foo import (x,)`
last_symbol = as_names
parent = last_symbol.parent
as_names = pytree.Node(298, [])
parent.set_child(-2, as_names)
as_names.append_child(last_symbol)
as_names.append_child(pytree.Leaf(token.COMMA, ","))
elif not hasattr(as_names.children[-1], "value") or as_names.children[-1].value != ",":
# `from foo import (x, y)` to `from foo import (x, y,)`
as_names.children.append(pytree.Leaf(token.COMMA, ","))
self.DefaultNodeVisit(node)
def transform(self, node, results):
assert results
syms = self.syms
anchor = results["anchor"]
# There are 2 types of nodes in the AST - Node and Leaf.
# Leaf nodes have a prefix and suffix that are meant for whitespaces and comments.
# It usually suffices to use the prefix only as the prefix of a node is the suffix
# of its previous node.
prefix = node.prefix
# before is the identifier that precedes the '.' before the 'None'.
before = [n.clone() for n in results["before"]]
if len(before) == 1:
before = before[0]
else:
before = pytree.Node(syms.power, before)
noneKeywd = String(repr("None"))
l_args = [before, Comma(), noneKeywd]
if l_args:
l_args[0].prefix = u""
l_args[2].prefix = u" "
new = Call(Name(u"getattr"), l_args)
new.prefix = prefix
return new
def transform(self, node, results):
local = results.get("local")
tail = results.get("tail")
if local:
local = local[0]
local_name = local.value
modules = self._names_to_modules[local_name]
if len(modules) > 1:
self.warnings(
node,
"Conflicting name '%s' is present in %s! Ignoring transformation!"
% local_name, modules)
return
module = list(modules)[0]
if all("module" not in res for res in self._import_matches):
self.warning(
node,
"Aggressive name matched '%s' but no corresponding import! Fix manually."
% local_name)
return
new_name = unicode(self.mapping[module + "." + local_name])
syms = self.syms
if tail:
tail = [t.clone() for t in tail]
# tail = tail.clone()
new = self.package_tree(new_name)
new = pytree.Node(syms.power, new + tail, prefix=local.prefix)
# Make sure the proper package is imported
package = new_name.rsplit(".", 1)[0]
touch_import(None, package, node)
return new
def _InsertPseudoParentheses(node):
comment_node = None
if isinstance(node, pytree.Node):
if node.children[-1].type == token.COMMENT:
comment_node = node.children[-1].clone()
node.children[-1].remove()
first = _GetFirstLeafNode(node)
last = _GetLastLeafNode(node)
lparen = pytree.Leaf(token.LPAR,
u'(',
context=('', (first.get_lineno(), first.column - 1)))
last_lineno = last.get_lineno()
if last.type == token.STRING and '\n' in last.value:
last_lineno += last.value.count('\n')
if last.type == token.STRING and '\n' in last.value:
last_column = len(last.value.split('\n')[-1]) + 1
else:
last_column = last.column + len(last.value) + 1
rparen = pytree.Leaf(token.RPAR,
u')',
context=('', (last_lineno, last_column)))
lparen.is_pseudo = True
rparen.is_pseudo = True
if isinstance(node, pytree.Node):
node.insert_child(0, lparen)
node.append_child(rparen)
if comment_node:
node.append_child(comment_node)
else:
new_node = pytree.Node(syms.atom, [lparen, node.clone(), rparen])
node.replace(new_node)
def test_node_repr(self):
l1 = pytree.Leaf(100, 'foo')
l2 = pytree.Leaf(100, 'bar', context=(' ', (1, 0)))
n1 = pytree.Node(1000, [l1, l2])
self.assertEqual(repr(n1),
'Node(1000, [%s, %s])' % (repr(l1), repr(l2)))
def test_node(self):
l1 = pytree.Leaf(100, 'foo')
l2 = pytree.Leaf(200, 'bar')
n1 = pytree.Node(1000, [l1, l2])
self.assertEqual(n1.type, 1000)
self.assertEqual(n1.children, [l1, l2])
def test_node_equality(self):
n1 = pytree.Node(1000, ())
n2 = pytree.Node(1000, [], context=(' ', (1, 0)))
self.assertEqual(n1, n2)
n3 = pytree.Node(1001, ())
self.assertNotEqual(n1, n3)
def test_node_str(self):
l1 = pytree.Leaf(100, "foo")
l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0)))
n1 = pytree.Node(1000, [l1, l2])
self.assertEqual(str(n1), "foo bar")
def test_node_repr(self):
l1 = pytree.Leaf(100, "foo")
l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0)))
n1 = pytree.Node(1000, [l1, l2])
self.assertEqual(repr(n1),
"Node(1000, [%s, %s])" % (repr(l1), repr(l2)))
def test_pre_order(self):
l1 = pytree.Leaf(100, "foo")
l2 = pytree.Leaf(100, "bar")
n1 = pytree.Node(1000, [l1, l2])
self.assertEqual(list(n1.pre_order()), [n1, l1, l2])
def transform(self, node, results):
name = results.get("name")
binding = results.get("binding")
pre = results.get("pre")
post = results.get("post")
simple = results.get("simple")
if simple:
binding = find_binding("intern", find_root(node), "sys")
binding.remove()
return
if binding:
if not pre and not post:
new_binding = find_binding("intern", find_root(node), "sys")
new_binding.remove()
return
elif not pre and post:
for ch in node.children:
if type(ch) == pytree.Node:
assert ch.children[0].prefix + "intern" \
== str(ch.children[0])
ch.children[0].remove() # intern
assert ch.children[0].prefix + "," \
== str(ch.children[0])
ch.children[0].remove() # ,
return
elif not post and pre:
for ch in node.children:
if type(ch) == pytree.Node:
assert ch.children[-1].prefix + "intern" \
== str(ch.children[-1])
ch.children[-1].remove() # intern
assert ch.children[-1].prefix + "," \
== str(ch.children[-1])
ch.children[-1].remove() # ,
return
elif post and pre:
for ch in node.children:
if type(ch) == pytree.Node:
for ch_ in ch.children:
if ch_ and ch_.prefix + "intern" == str(ch_):
last_ch_ = ch_.prev_sibling
ch_.remove() # intern
assert last_ch_.prefix + "," \
== str(last_ch_)
last_ch_.remove() # ,
return
syms = self.syms
obj = results["obj"].clone()
if obj.type == syms.arglist:
newarglist = obj.clone()
else:
newarglist = pytree.Node(syms.arglist, [obj.clone()])
after = results["after"]
if after:
after = [n.clone() for n in after]
new = pytree.Node(syms.power, [Name("intern")] + [
pytree.Node(syms.trailer, [
results["lpar"].clone(), newarglist, results["rpar"].clone()
] + after)
])
new.prefix = node.prefix
return new
def testNodeNameForNode(self):
leaf = pytree.Leaf(token.LPAR, '(')
node = pytree.Node(pygram.python_grammar.symbol2number['suite'], [leaf])
self.assertEqual('suite', pytree_utils.NodeName(node))
def test_node_str(self):
l1 = pytree.Leaf(100, 'foo')
l2 = pytree.Leaf(100, 'bar', context=(' ', (1, 0)))
n1 = pytree.Node(1000, [l1, l2])
self.assertEqual(str(n1), 'foo bar')
def setUp(self):
self._leaf = pytree.Leaf(token.LPAR, '(')
self._node = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'],
[pytree.Leaf(token.NAME, 'foo')])
def _InsertPseudoParentheses(node):
"""Insert pseudo parentheses so that dicts can be formatted correctly."""
comment_node = None
if isinstance(node, pytree.Node):
if node.children[-1].type == token.COMMENT:
comment_node = node.children[-1].clone()
node.children[-1].remove()
first = pytree_utils.FirstLeafNode(node)
last = pytree_utils.LastLeafNode(node)
if first == last and first.type == token.COMMENT:
# A comment was inserted before the value, which is a pytree.Leaf.
# Encompass the dictionary's value into an ATOM node.
last = first.next_sibling
last_clone = last.clone()
new_node = pytree.Node(syms.atom, [first.clone(), last_clone])
for orig_leaf, clone_leaf in zip(last.leaves(), last_clone.leaves()):
pytree_utils.CopyYapfAnnotations(orig_leaf, clone_leaf)
if hasattr(orig_leaf, 'is_pseudo'):
clone_leaf.is_pseudo = orig_leaf.is_pseudo
node.replace(new_node)
node = new_node
last.remove()
first = pytree_utils.FirstLeafNode(node)
last = pytree_utils.LastLeafNode(node)
lparen = pytree.Leaf(token.LPAR,
u'(',
context=('', (first.get_lineno(), first.column - 1)))
last_lineno = last.get_lineno()
if last.type == token.STRING and '\n' in last.value:
last_lineno += last.value.count('\n')
if last.type == token.STRING and '\n' in last.value:
last_column = len(last.value.split('\n')[-1]) + 1
else:
last_column = last.column + len(last.value) + 1
rparen = pytree.Leaf(token.RPAR,
u')',
context=('', (last_lineno, last_column)))
lparen.is_pseudo = True
rparen.is_pseudo = True
if isinstance(node, pytree.Node):
node.insert_child(0, lparen)
node.append_child(rparen)
if comment_node:
node.append_child(comment_node)
_AppendFirstLeafTokenSubtype(node,
format_token.Subtype.DICTIONARY_VALUE)
else:
clone = node.clone()
for orig_leaf, clone_leaf in zip(node.leaves(), clone.leaves()):
pytree_utils.CopyYapfAnnotations(orig_leaf, clone_leaf)
new_node = pytree.Node(syms.atom, [lparen, clone, rparen])
node.replace(new_node)
_AppendFirstLeafTokenSubtype(clone,
format_token.Subtype.DICTIONARY_VALUE)
