def transform(self, node, results):
result = super().transform(node, results)
if not utils.is_listcomp(result):
# Keep performance improvement from six.moves.filter in iterator
# contexts on Python 2.7.
fixer_util.touch_import("six.moves", "filter", node)
return result
def transform_iter(self, node, results):
"""Call six.(iter|view)items() and friends."""
# Make sure six is imported.
fixer_util.touch_import(None, "six", node)
# Copy of self.transform() from fissix.fix_dict with some changes to
# use the six.* methods.
head = results["head"]
method = results["method"][0] # Extract node for method name
tail = results["tail"]
syms = self.syms
method_name = method.value
name = fixer_util.Name("six." + method_name, prefix=node.prefix)
assert method_name.startswith(("iter", "view")), repr(method)
assert method_name[4:] in ("keys", "items", "values"), repr(method)
head = [n.clone() for n in head]
tail = [n.clone() for n in tail]
new = pytree.Node(syms.power, head)
new.prefix = ""
new = fixer_util.Call(name, [new])
if tail:
new = pytree.Node(syms.power, [new] + tail)
new.prefix = node.prefix
return new
def transform(self, node, results):
if _literal_re.match(node.value):
fixer_util.touch_import(None, "six", node)
new = node.clone()
new.value = new.value[1:]
new.prefix = ""
node.replace(Call(Name("six.u", prefix=node.prefix), [new]))
def transform(self, node, results):
result = super().transform(node, results)
if not utils.is_listcomp(result):
# Always use the import even if no change is required so as to have
# improved performance in iterator contexts even on Python 2.7.
fixer_util.touch_import("six.moves", "map", node)
return result
def _add_import(node: LN, capture: Capture, filename: Filename):
if node.type != python_symbols.file_input:
return
if filename in paddle_imported:
return
if filename in paddle_found:
touch_import(None, 'paddle', node, force=True)
log_info(filename, node.get_lineno(), 'add "import paddle"')
paddle_imported.add(filename)
def rewrite_decorator(node, decorator_name, marker):
"""
Replaces usage of ``@<decorator_name>`` with ``@<marker>``
"""
decorator = get_decorator(node, decorator_name, marker)
# pylint: disable=no-member
for leaf in decorator.children:
if leaf.type == TOKEN.AT:
continue
if leaf.type == TOKEN.NAME and leaf.value == decorator_name:
leaf.value = marker
continue
if leaf.type == syms.atom:
# This scenario is func(*args)
#
# Sometimes instead of passing expanded arguments 'foo(arg1, arg2)' a tuple or a
# list is passed as first argument 'foo([arg1, arg2])' or 'foo((arg1, arg2)).
# Let's break out of those containers.
break_out_of_atom = False
for atom_child in leaf.children:
if atom_child.type not in (syms.listmaker, syms.dictsetmaker,
syms.testlist_gexp):
atom_child.remove()
continue
break_out_of_atom = True
break
if break_out_of_atom:
continue
elif leaf.type == syms.arglist:
# This scenario is func(*args, **kwargs)
#
for arg_child in leaf.children:
if arg_child.type == syms.atom:
# Sometimes instead of passing expanded arguments 'foo(arg1, arg2)' a tuple or a
# list is passed as first argument 'foo([arg1, arg2])' or 'foo((arg1, arg2)).
# Let's break out of those containers.
break_out_of_atom = False
for atom_child in arg_child.children:
if atom_child.type not in (
syms.listmaker,
syms.dictsetmaker,
syms.testlist_gexp,
):
atom_child.remove()
continue
break_out_of_atom = True
break
if break_out_of_atom:
continue
# pylint: enable=no-member
touch_import(None, "pytest", node)
remove_from_import(node, "tests.support.helpers", decorator_name)
def gdaltest_other_skipfails(node, capture, filename):
"""
Replaces a generic call to `return 'skip'` or `return 'fail'`,
with a `pytest.skip()` or `pytest.fail()`.
If there's a call to `gdal.post_reason()` immediately preceding
the return statement, uses that reason as the argument to the
skip/fail function.
Ignores/preserves print() calls between the two.
Examples:
gdal.post_reason('foo')
print('everything is broken')
return 'fail'
-->
print('everything is broken')
pytest.fail('foo')
gdal.post_reason('foo')
return 'skip'
-->
pytest.skip('foo')
return 'skip'
-->
pytest.skip()
"""
if flags["debug"]:
print(f"expression: {capture}")
returntype = string_value(capture["returntype"])
if returntype not in ("skip", "fail"):
return
reason = None
if capture.get('post_reason_call'):
# Remove the gdal.post_reason() statement altogether. Preserve whitespace
reason = capture["reason"].clone()
prefix = capture["post_reason_call"].prefix
next_node = capture["post_reason_call"].next_sibling
capture["post_reason_call"].remove()
next_node.prefix = prefix
# Replace the return statement with a call to pytest.skip() or pytest.fail().
# Include the reason message if there was one.
replacement = Attr(kw("pytest", prefix=capture["return_call"].prefix),
kw(returntype, prefix="")) + [ArgList(listify(reason))]
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
capture["return_call"].replace(replacement)
def transform(self, node, results):
exc = results["exc"].clone()
val = results["val"].clone()
tb = results["tb"].clone()
exc.prefix = ""
val.prefix = tb.prefix = " "
fixer_util.touch_import(None, "six", node)
return Call(
Name("six.reraise"), [exc, Comma(), val, Comma(), tb], prefix=node.prefix
)
def transform(self, node, results):
if self.should_skip(node):
return
fixer_util.touch_import("six.moves", "input", node)
name = results["name"]
if name.value == "raw_input":
name.replace(Name("input", prefix=name.prefix))
else:
new_node = node.clone()
new_node.prefix = ""
return Call(Name("eval"), [new_node], prefix=node.prefix)
def transform(self, node, results):
imports = results["imports"]
if imports.type == syms.import_as_name or not imports.children:
children = [imports]
else:
children = imports.children
for child in children[::2]:
if child.type == token.NAME:
name_node = child
elif child.type == token.STAR:
# Just leave the import as is.
return
else:
assert child.type == syms.import_as_name
name_node = child.children[0]
member_name = name_node.value
if member_name in (
"imap",
"izip",
"ifilter",
"ifilterfalse",
"izip_longest",
):
child.value = None
fixer_util.touch_import("six.moves", member_name[1:], node)
child.remove()
# Make sure the import statement is still sane
children = imports.children[:] or [imports]
remove_comma = True
for child in children:
if remove_comma and child.type == token.COMMA:
child.remove()
else:
remove_comma ^= True
while children and children[-1].type == token.COMMA:
children.pop().remove()
# If there are no imports left, just get rid of the entire statement
if (
not (imports.children or getattr(imports, "value", None))
or imports.parent is None
):
p = node.prefix
node = BlankLine()
node.prefix = p
return node
def handle_assertraises(node, capture, arguments):
"""
with self.assertRaises(x):
--> with pytest.raises(x):
self.assertRaises(ValueError, func, arg1)
--> pytest.raises(ValueError, func, arg1)
"""
capture['attr1'].replace(kw('pytest', prefix=capture['attr1'].prefix))
capture['attr2'].replace(
Node(syms.trailer, [Dot(), kw('raises', prefix='')]))
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
def handle_assert_regex(node, capture, arguments):
"""
self.assertRegex(text, pattern, msg)
--> assert re.search(pattern, text), msg
self.assertNotRegex(text, pattern, msg)
--> assert not re.search(pattern, text), msg
"""
function_name = capture["function_name"].value
invert = function_name in INVERT_FUNCTIONS
function_name = SYNONYMS.get(function_name, function_name)
num_arguments = ARGUMENTS[function_name]
if len(arguments) not in (num_arguments, num_arguments + 1):
# Not sure what this is. Leave it alone.
return None
if len(arguments) == num_arguments:
message = None
else:
message = arguments.pop()
if message.type == syms.argument:
# keyword argument (e.g. `msg=abc`)
message = message.children[2].clone()
arguments[0].prefix = " "
arguments[1].prefix = ""
# Adds a 'import re' if there wasn't one already
touch_import(None, "re", node)
op_tokens = []
if invert:
op_tokens.append(keyword("not"))
assert_test_nodes = [
Node(
syms.power,
op_tokens + Attr(keyword("re"), keyword("search", prefix="")) +
[ArgList([arguments[1], Comma(), arguments[0]])],
)
]
return Assert(assert_test_nodes,
message.clone() if message else None,
prefix=node.prefix)
def transform(self, node, results):
prefix = None
func = results["func"][0]
if "it" in results and func.value not in ("ifilterfalse",
"izip_longest"):
dot, it = (results["dot"], results["it"])
# Remove the 'itertools'
prefix = it.prefix
it.remove()
# Replace the node wich contains ('.', 'function') with the
# function (to be consistant with the second part of the pattern)
dot.remove()
func.parent.replace(func)
fixer_util.touch_import("six.moves", func.value[1:], node)
prefix = prefix or func.prefix
func.replace(Name(func.value[1:], prefix=prefix))
def pytest_approx(node, capture, filename):
target_value = listify(capture['target_value'])[0].clone()
target_value.prefix = ''
abs_tolerance = capture['abs_tolerance'].clone()
abs_tolerance.prefix = ''
op_value = listify(capture['op'])[0].value
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
if op_value in ('<', '<='):
# as you'd expect in an assert statement
operator = Leaf(TOKEN.EQEQUAL, '==', prefix=' ')
else:
# probably in an if statement
operator = Leaf(TOKEN.NOTEQUAL, '!=', prefix=' ')
node.replace(
Node(
syms.comparison,
[
capture['lhs'].clone(),
operator,
Node(
syms.power,
[
kw('pytest'),
Node(
syms.trailer,
[Leaf(TOKEN.DOT, ".", prefix=''), kw('approx', prefix='')],
prefix='',
),
ArgList(
[
target_value,
Comma(),
KeywordArg(kw('abs'), abs_tolerance),
]
),
],
),
],
prefix=node.prefix,
)
)
def replace_unicode_methods(node, capture, arguments):
# remove any existing __str__ method
b = find_binding("__str__", capture['suite'])
if b and b.type == syms.funcdef:
b.remove()
# rename __unicode__ to __str__
funcname = capture['funcname'].clone()
funcname.value = '__str__'
capture['funcname'].replace(funcname)
# Add a six import
touch_import(None, "six", node)
# Decorate the class with `@six.python_2_unicode_compatible`
classdef = node.clone()
classdef.prefix = ''
decorated = Node(
syms.decorated,
[
Node(
syms.decorator,
[
Leaf(TOKEN.AT, '@', prefix=node.prefix),
Node(
syms.dotted_name,
[
Name('six'),
Dot(),
Name('python_2_unicode_compatible')
],
),
Newline(),
],
prefix=node.prefix,
),
classdef,
],
prefix=node.prefix,
)
node.replace(decorated)
def handle_assertraises(node, capture, arguments):
"""
with self.assertRaises(x):
--> with pytest.raises(x):
self.assertRaises(ValueError, func, arg1)
--> pytest.raises(ValueError, func, arg1)
"""
capture["self_attr"].replace(
keyword("pytest", prefix=capture["self_attr"].prefix))
capture["raises_attr"].replace(
Node(syms.trailer, [Dot(), keyword("raises", prefix="")]))
# Let's remove the msg= keyword argument if found
for child in node.children:
if child.type != syms.trailer:
continue
for tchild in child.children:
if tchild.type != syms.arglist:
continue
previous_argument = None
for argument in tchild.children:
if isinstance(argument, Leaf):
previous_argument = argument
continue
if isinstance(argument, Node):
if argument.type != syms.argument:
previous_argument = argument
continue
for leaf in argument.leaves():
if leaf.value == "msg":
argument.remove()
if previous_argument.value == ",":
# previous_argument is a comma, remove it.
previous_argument.remove()
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
def test_name_import(self):
node = parse("bar()")
fixer_util.touch_import(None, "cgi", node)
self.assertEqual(str(node), "import cgi\nbar()\n\n")
def test_from_import(self):
node = parse("bar()")
fixer_util.touch_import("html", "escape", node)
self.assertEqual(str(node), "from html import escape\nbar()\n\n")
def test_beginning(self):
node = parse("bar()")
fixer_util.touch_import(None, "foo", node)
self.assertEqual(str(node), "import foo\nbar()\n\n")
def test_after_imports(self):
node = parse('"""foo"""\nimport bar\nbar()')
fixer_util.touch_import(None, "foo", node)
self.assertEqual(str(node), '"""foo"""\nimport bar\nimport foo\nbar()\n\n')
def transform(self, node, results):
touch_import("functools", "reduce", node)
def transform(self, node, results):
if self.should_skip(node):
return
fixer_util.touch_import("six.moves", "range", node)
return super().transform(node, results)
def transform(self, node, results):
if not has_metaclass(node):
return # pragma: no cover
fixup_parse_tree(node)
# find metaclasses, keep the last one
last_metaclass = None
for suite, i, stmt in find_metas(node):
last_metaclass = stmt
stmt.remove()
text_type = node.children[0].type # always Leaf(nnn, 'class')
# figure out what kind of classdef we have
if len(node.children) == 7:
# Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite])
# 0 1 2 3 4 5 6
if node.children[3].type == syms.arglist:
arglist = node.children[3]
# Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite])
else:
parent = node.children[3].clone()
arglist = Node(syms.arglist, [parent])
node.set_child(3, arglist)
elif len(node.children) == 6:
# Node(classdef, ['class', 'name', '(', ')', ':', suite])
# 0 1 2 3 4 5
arglist = Node(syms.arglist, [])
node.insert_child(3, arglist)
elif len(node.children) == 4:
# Node(classdef, ['class', 'name', ':', suite])
# 0 1 2 3
arglist = Node(syms.arglist, [])
node.insert_child(2, Leaf(token.RPAR, ")"))
node.insert_child(2, arglist)
node.insert_child(2, Leaf(token.LPAR, "("))
else:
raise ValueError("Unexpected class definition") # pragma: no cover
fixer_util.touch_import(None, "six", node)
metaclass = last_metaclass.children[0].children[2].clone()
metaclass.prefix = ""
arguments = [metaclass]
if arglist.children:
bases = arglist.clone()
bases.prefix = " "
arguments.extend([Comma(), bases])
arglist.replace(
Call(Name("six.with_metaclass", prefix=arglist.prefix), arguments)
)
fixup_indent(suite)
# check for empty suite
if not suite.children:
# one-liner that was just __metaclass__
suite.remove()
pass_leaf = Leaf(text_type, "pass")
pass_leaf.prefix = last_metaclass.prefix
node.append_child(pass_leaf)
node.append_child(Leaf(token.NEWLINE, "\n"))
elif len(suite.children) > 1 and (
suite.children[-2].type == token.INDENT
and suite.children[-1].type == token.DEDENT
):
# there was only one line in the class body and it was __metaclass__
pass_leaf = Leaf(text_type, "pass")
suite.insert_child(-1, pass_leaf)
suite.insert_child(-1, Leaf(token.NEWLINE, "\n"))
def transform(self, node, results):
fixer_util.touch_import(None, "six", node)
pair = results["pair"]
pair.replace(fixer_util.Name("six.integer_types", prefix=pair.prefix))
def assertalmostequal_to_assert(node, capture, arguments):
function_name = capture["function_name"].value
invert = function_name in INVERT_FUNCTIONS
function_name = SYNONYMS.get(function_name, function_name)
nargs = len(arguments)
if nargs < 2 or nargs > 5:
return None
def get_kwarg_value(index, name):
idx = 0
for arg in arguments:
if arg.type == syms.argument:
if arg.children[0].value == name:
return arg.children[2].clone()
else:
if idx == index:
return arg.clone()
idx += 1
return None
first = get_kwarg_value(0, "first")
second = get_kwarg_value(1, "second")
if first is None or second is None:
# Not sure what this is, leave it alone
return
places = get_kwarg_value(2, "places")
msg = get_kwarg_value(3, "msg")
delta = get_kwarg_value(4, "delta")
if delta is not None:
try:
abs_delta = float(delta.value)
except ValueError:
# this should be a number, give up.
return
else:
if places is None:
places = 7
else:
try:
places = int(places.value)
except (ValueError, AttributeError):
# this should be an int, give up.
return
abs_delta = "1e-%d" % places
arguments[1].prefix = ""
if invert:
op_token = Leaf(TOKEN.NOTEQUAL, "!=", prefix=" ")
else:
op_token = Leaf(TOKEN.EQEQUAL, "==", prefix=" ")
assert_test_nodes = [
Node(
syms.comparison,
[
arguments[0],
op_token,
Node(
syms.power,
Attr(keyword("pytest"), keyword("approx", prefix="")) + [
ArgList([
arguments[1],
Comma(),
KeywordArg(keyword("abs"),
Leaf(TOKEN.NUMBER, abs_delta)),
])
],
),
],
)
]
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
return Assert(assert_test_nodes,
msg.clone() if msg else None,
prefix=node.prefix)
def assertmethod_to_assert(node, capture, arguments):
"""
self.assertEqual(foo, bar, msg)
--> assert foo == bar, msg
self.assertTrue(foo, msg)
--> assert foo, msg
self.assertIsNotNone(foo, msg)
--> assert foo is not None, msg
.. etc
"""
function_name = capture["function_name"].value
invert = function_name in INVERT_FUNCTIONS
function_name = SYNONYMS.get(function_name, function_name)
num_arguments = ARGUMENTS[function_name]
if len(arguments) not in (num_arguments, num_arguments + 1):
# Not sure what this is. Leave it alone.
return None
if len(arguments) == num_arguments:
message = None
else:
message = arguments.pop()
if message.type == syms.argument:
# keyword argument (e.g. `msg=abc`)
message = message.children[2].clone()
if function_name == "assertIsInstance":
arguments[0].prefix = ""
assert_test_nodes = [
Call(keyword("isinstance"),
[arguments[0], Comma(), arguments[1]])
]
if invert:
assert_test_nodes.insert(0, keyword("not"))
elif function_name == "assertAlmostEqual":
arguments[1].prefix = ""
# TODO: insert the `import pytest` at the top of the file
if invert:
op_token = Leaf(TOKEN.NOTEQUAL, "!=", prefix=" ")
else:
op_token = Leaf(TOKEN.EQEQUAL, "==", prefix=" ")
assert_test_nodes = [
Node(
syms.comparison,
[
arguments[0],
op_token,
Node(
syms.power,
Attr(keyword("pytest"), keyword("approx", prefix="")) +
[
ArgList([
arguments[1],
Comma(),
KeywordArg(keyword("abs"),
Leaf(TOKEN.NUMBER, "1e-7")),
])
],
),
],
)
]
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
else:
op_tokens = OPERATORS[function_name]
if not isinstance(op_tokens, list):
op_tokens = [op_tokens]
op_tokens = [o.clone() for o in op_tokens]
if invert:
if not op_tokens:
op_tokens.append(keyword("not"))
elif op_tokens[0].type == TOKEN.NAME and op_tokens[0].value == "is":
op_tokens[0] = Node(
syms.comp_op,
[keyword("is"), keyword("not")], prefix=" ")
elif op_tokens[0].type == TOKEN.NAME and op_tokens[0].value == "in":
op_tokens[0] = Node(
syms.comp_op,
[keyword("not"), keyword("in")], prefix=" ")
elif op_tokens[0].type == TOKEN.EQEQUAL:
op_tokens[0] = Leaf(TOKEN.NOTEQUAL, "!=", prefix=" ")
if num_arguments == 2:
# a != b, etc.
assert_test_nodes = [arguments[0]] + op_tokens + [arguments[1]]
elif function_name == "assertTrue":
assert_test_nodes = op_tokens + [arguments[0]]
# not a
elif function_name == "assertIsNone":
# a is not None
assert_test_nodes = [arguments[0]] + op_tokens
return Assert(assert_test_nodes,
message.clone() if message else None,
prefix=node.prefix)
def process_class(node: LN, capture: Capture,
filename: Filename) -> Optional[LN]:
"""Do the processing/modification of the class node"""
print(f"Class for show(): {filename}:{node.get_lineno()}")
suite = get_child(node, python_symbols.suite)
# Get the suite indent
indent = find_indentation(suite)
# Find the show() function
funcs = {
x.children[1].value: x
for x in get_children(suite, python_symbols.funcdef)
}
show_func = funcs["show"]
# Get the name of the filename object keyword that this show uses
show_keyword = analyse_show(show_func, filename)
if show_keyword is None:
# show() writes to stdout. Use Graeme's method for now.
kludge_text = (
f"def __str__(self):\n{indent} return kludge_show_to_str(self)\n\n"
)
else:
# We can more intelligently call show
kludge_text = (f"def __str__(self):\n{indent} out = StringIO()\n"
f"{indent} self.show({show_keyword}=out)\n"
f"{indent} return out.getvalue.rstrip()\n\n")
# To avoid having to work out indent correction, just generate with correct
kludge_node = get_child(driver.parse_string(kludge_text),
python_symbols.funcdef)
# Work out if we have any trailing text/comments that need to be moved
trail_node = get_trailing_text_node(suite)
pre, post = split_suffix(trail_node)
# The trailing contents of this node will be moved
trail_node.prefix = pre
# Get the dedent node at the end of the previous - suite always ends with dedent
# This is the dedent before the end of the suite, so the one to alter for the new
# function
# If we aren't after a suite then we don't have a DEDENT so don't need to
# correct the indentation.
# children[-2] is the last statement at the end of the suite
# children[-1] is the suite on a function definition
# children[-1] is the dedent at the end of the function's suite
if suite.children[-2].type == python_symbols.funcdef:
last_func_dedent_node = suite.children[-2].children[-1].children[-1]
last_func_dedent_node.prefix += "\n" + indent
suite.children.insert(-1, kludge_node)
# Get the kludge dedent - now the last dedent
kludge_dedent = kludge_node.children[-1].children[-1]
kludge_dedent.prefix = post
# Make sure the functions used are available
if show_keyword is None:
touch_import("libtbx.utils", "kludge_show_to_str", node)
else:
touch_import("six.moves", "StringIO", node)
def transform(self, node, results):
result = super().transform(node, results)
# Always use six.moves.zip so that even Python 2.7 gets performance
# boost from using itertools in iterator contexts.
fixer_util.touch_import("six.moves", "zip", node)
return result
def touch_import(package, name, node):
fixer_util.touch_import(package, name, node)
def transform(self, node, results):
fixer_util.touch_import(None, "six", node)
return fixer_util.Name("six.text_type", prefix=node.prefix)
