def gdaltest_other_skipfails(node, capture, filename):
"""
Replaces a generic call to `return 'skip'` or `return 'fail'`,
with a `pytest.skip()` or `pytest.fail()`.
If there's a call to `gdal.post_reason()` immediately preceding
the return statement, uses that reason as the argument to the
skip/fail function.
Ignores/preserves print() calls between the two.
Examples:
gdal.post_reason('foo')
print('everything is broken')
return 'fail'
-->
print('everything is broken')
pytest.fail('foo')
gdal.post_reason('foo')
return 'skip'
-->
pytest.skip('foo')
return 'skip'
-->
pytest.skip()
"""
if flags["debug"]:
print(f"expression: {capture}")
returntype = string_value(capture["returntype"])
if returntype not in ("skip", "fail"):
return
reason = None
if capture.get('post_reason_call'):
# Remove the gdal.post_reason() statement altogether. Preserve whitespace
reason = capture["reason"].clone()
prefix = capture["post_reason_call"].prefix
next_node = capture["post_reason_call"].next_sibling
capture["post_reason_call"].remove()
next_node.prefix = prefix
# Replace the return statement with a call to pytest.skip() or pytest.fail().
# Include the reason message if there was one.
replacement = Attr(kw("pytest", prefix=capture["return_call"].prefix),
kw(returntype, prefix="")) + [ArgList(listify(reason))]
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
capture["return_call"].replace(replacement)
def handle_assert_regex(node, capture, arguments):
"""
self.assertRegex(text, pattern, msg)
--> assert re.search(pattern, text), msg
self.assertNotRegex(text, pattern, msg)
--> assert not re.search(pattern, text), msg
"""
function_name = capture["function_name"].value
invert = function_name in INVERT_FUNCTIONS
function_name = SYNONYMS.get(function_name, function_name)
num_arguments = ARGUMENTS[function_name]
if len(arguments) not in (num_arguments, num_arguments + 1):
# Not sure what this is. Leave it alone.
return None
if len(arguments) == num_arguments:
message = None
else:
message = arguments.pop()
if message.type == syms.argument:
# keyword argument (e.g. `msg=abc`)
message = message.children[2].clone()
arguments[0].prefix = " "
arguments[1].prefix = ""
# Adds a 'import re' if there wasn't one already
touch_import(None, "re", node)
op_tokens = []
if invert:
op_tokens.append(keyword("not"))
assert_test_nodes = [
Node(
syms.power,
op_tokens + Attr(keyword("re"), keyword("search", prefix="")) +
[ArgList([arguments[1], Comma(), arguments[0]])],
)
]
return Assert(assert_test_nodes,
message.clone() if message else None,
prefix=node.prefix)
def rewrite_print(node, capture, filename):
"""Given a print node, rewrite to a logger.debug"""
params = capture["function_parameters"]
# Args is either a Leaf or an arglist Node here
arglist = params.children[1]
# Extract the arguments from this list
if isinstance(arglist, Node) and arglist.type == syms.arglist:
args = [x for x in arglist.children if not x.type == token.COMMA]
# Remove kwargs for now
non_kwargs = [x for x in args if not x.type == syms.argument]
# Read out things like sep here
sep = " "
if not len(non_kwargs) == len(args):
first_leaf = next(node.leaves())
print(
"Warning: {}:{} Not handling keyword argument loggers".format(
filename, first_leaf.lineno))
return None
# If more than one child, we need to construct a new joiner string
if len(non_kwargs) > 1:
# Instead of placing a new, modify the first if it's a string
if arglist.children[0].type == token.STRING:
value = arglist.children[0].value
last_char = value[-1]
new_end = sep + sep.join(["%s"] * (len(non_kwargs) - 1))
arglist.children[0].value = value[:-1] + new_end + last_char
else:
arglist.insert_child(
0, String('"' + sep.join(["%s"] * len(non_kwargs)) + '"'))
arglist.insert_child(1, Comma())
arglist.children[2].prefix = " " + arglist.children[2].prefix
# Use the possibly rewritten parameters in the new call
new_node = Attr(Name("logger"), Name("debug"))
new_node[0].prefix = node.children[0].prefix
node.children = new_node + [params]
def modify_dict_literal(node: LN, capture: Capture,
filename: Filename) -> Optional[LN]:
toks = iter(capture.get("body"))
items = []
prefix = ""
while True:
try:
tok = next(toks)
if tok.type == TOKEN.DOUBLESTAR:
body = next(toks).clone()
body.prefix = prefix + tok.prefix + body.prefix
items.append(body)
else:
colon = next(toks)
value = next(toks).clone()
value.prefix = colon.prefix + value.prefix
if items and isinstance(items[-1], list):
items[-1].append(TupleNode(tok, value))
else:
items.append([TupleNode(tok, value)])
comma = next(toks)
prefix = comma.prefix
except StopIteration:
break
listitems = []
for item in items:
if listitems:
listitems.extend([Space(), Plus(), Space()])
if isinstance(item, list):
listitems.append(ListNode(*item))
else:
call = Node(SYMBOL.test, [*Attr(item, Name("items")), ArgList([])])
listitems.append(call)
args = listitems
if len(listitems) > 1:
args = [Node(SYMBOL.arith_expr, args)]
args.append(String(node.children[-1].prefix))
return Call(Name("dict"), args, prefix=node.prefix)
def transform(self, node, results):
# First, find the sys import. We'll just hope it's global scope.
if "sys_import" in results:
if self.sys_import is None:
self.sys_import = results["sys_import"]
return
func = results["func"].clone()
func.prefix = ""
register = pytree.Node(syms.power, Attr(Name("atexit"), Name("register")))
call = Call(register, [func], node.prefix)
node.replace(call)
if self.sys_import is None:
# That's interesting.
self.warning(
node,
"Can't find sys import; Please add an atexit "
"import at the top of your file.",
)
return
# Now add an atexit import after the sys import.
names = self.sys_import.children[1]
if names.type == syms.dotted_as_names:
names.append_child(Comma())
names.append_child(Name("atexit", " "))
else:
containing_stmt = self.sys_import.parent
position = containing_stmt.children.index(self.sys_import)
stmt_container = containing_stmt.parent
new_import = pytree.Node(
syms.import_name, [Name("import"), Name("atexit", " ")]
)
new = pytree.Node(syms.simple_stmt, [new_import])
containing_stmt.insert_child(position + 1, Newline())
containing_stmt.insert_child(position + 2, new)
def test_returns(self):
attr = Attr(Name("a"), Name("b"))
self.assertEqual(type(attr), list)
def test(self):
call = parse("foo()", strip_levels=2)
self.assertStr(Attr(Name("a"), Name("b")), "a.b")
self.assertStr(Attr(call, Name("b")), "foo().b")
def assertalmostequal_to_assert(node, capture, arguments):
function_name = capture["function_name"].value
invert = function_name in INVERT_FUNCTIONS
function_name = SYNONYMS.get(function_name, function_name)
nargs = len(arguments)
if nargs < 2 or nargs > 5:
return None
def get_kwarg_value(index, name):
idx = 0
for arg in arguments:
if arg.type == syms.argument:
if arg.children[0].value == name:
return arg.children[2].clone()
else:
if idx == index:
return arg.clone()
idx += 1
return None
first = get_kwarg_value(0, "first")
second = get_kwarg_value(1, "second")
if first is None or second is None:
# Not sure what this is, leave it alone
return
places = get_kwarg_value(2, "places")
msg = get_kwarg_value(3, "msg")
delta = get_kwarg_value(4, "delta")
if delta is not None:
try:
abs_delta = float(delta.value)
except ValueError:
# this should be a number, give up.
return
else:
if places is None:
places = 7
else:
try:
places = int(places.value)
except (ValueError, AttributeError):
# this should be an int, give up.
return
abs_delta = "1e-%d" % places
arguments[1].prefix = ""
if invert:
op_token = Leaf(TOKEN.NOTEQUAL, "!=", prefix=" ")
else:
op_token = Leaf(TOKEN.EQEQUAL, "==", prefix=" ")
assert_test_nodes = [
Node(
syms.comparison,
[
arguments[0],
op_token,
Node(
syms.power,
Attr(keyword("pytest"), keyword("approx", prefix="")) + [
ArgList([
arguments[1],
Comma(),
KeywordArg(keyword("abs"),
Leaf(TOKEN.NUMBER, abs_delta)),
])
],
),
],
)
]
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
return Assert(assert_test_nodes,
msg.clone() if msg else None,
prefix=node.prefix)
def assertmethod_to_assert(node, capture, arguments):
"""
self.assertEqual(foo, bar, msg)
--> assert foo == bar, msg
self.assertTrue(foo, msg)
--> assert foo, msg
self.assertIsNotNone(foo, msg)
--> assert foo is not None, msg
.. etc
"""
function_name = capture["function_name"].value
invert = function_name in INVERT_FUNCTIONS
function_name = SYNONYMS.get(function_name, function_name)
num_arguments = ARGUMENTS[function_name]
if len(arguments) not in (num_arguments, num_arguments + 1):
# Not sure what this is. Leave it alone.
return None
if len(arguments) == num_arguments:
message = None
else:
message = arguments.pop()
if message.type == syms.argument:
# keyword argument (e.g. `msg=abc`)
message = message.children[2].clone()
if function_name == "assertIsInstance":
arguments[0].prefix = ""
assert_test_nodes = [
Call(keyword("isinstance"),
[arguments[0], Comma(), arguments[1]])
]
if invert:
assert_test_nodes.insert(0, keyword("not"))
elif function_name == "assertAlmostEqual":
arguments[1].prefix = ""
# TODO: insert the `import pytest` at the top of the file
if invert:
op_token = Leaf(TOKEN.NOTEQUAL, "!=", prefix=" ")
else:
op_token = Leaf(TOKEN.EQEQUAL, "==", prefix=" ")
assert_test_nodes = [
Node(
syms.comparison,
[
arguments[0],
op_token,
Node(
syms.power,
Attr(keyword("pytest"), keyword("approx", prefix="")) +
[
ArgList([
arguments[1],
Comma(),
KeywordArg(keyword("abs"),
Leaf(TOKEN.NUMBER, "1e-7")),
])
],
),
],
)
]
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
else:
op_tokens = OPERATORS[function_name]
if not isinstance(op_tokens, list):
op_tokens = [op_tokens]
op_tokens = [o.clone() for o in op_tokens]
if invert:
if not op_tokens:
op_tokens.append(keyword("not"))
elif op_tokens[0].type == TOKEN.NAME and op_tokens[0].value == "is":
op_tokens[0] = Node(
syms.comp_op,
[keyword("is"), keyword("not")], prefix=" ")
elif op_tokens[0].type == TOKEN.NAME and op_tokens[0].value == "in":
op_tokens[0] = Node(
syms.comp_op,
[keyword("not"), keyword("in")], prefix=" ")
elif op_tokens[0].type == TOKEN.EQEQUAL:
op_tokens[0] = Leaf(TOKEN.NOTEQUAL, "!=", prefix=" ")
if num_arguments == 2:
# a != b, etc.
assert_test_nodes = [arguments[0]] + op_tokens + [arguments[1]]
elif function_name == "assertTrue":
assert_test_nodes = op_tokens + [arguments[0]]
# not a
elif function_name == "assertIsNone":
# a is not None
assert_test_nodes = [arguments[0]] + op_tokens
return Assert(assert_test_nodes,
message.clone() if message else None,
prefix=node.prefix)
def encapsulate_transform(
node: LN, capture: Capture, filename: Filename
) -> None:
if "attr_assignment" in capture:
leaf = capture["attr_name"]
leaf.replace(Name(new_name, prefix=leaf.prefix))
if make_property:
# TODO: capture and use type annotation from original assignment
class_node = get_class(node)
suite = find_first(class_node, SYMBOL.suite)
assert isinstance(suite, Node)
indent_node = find_first(suite, TOKEN.INDENT)
assert isinstance(indent_node, Leaf)
indent = indent_node.value
getter = Node(
SYMBOL.decorated,
[
Node(
SYMBOL.decorator,
[
Leaf(TOKEN.AT, "@"),
Name("property"),
Leaf(TOKEN.NEWLINE, "\n"),
],
),
Node(
SYMBOL.funcdef,
[
Name("def", indent),
Name(old_name, prefix=" "),
Node(
SYMBOL.parameters,
[LParen(), Name("self"), RParen()],
),
Leaf(TOKEN.COLON, ":"),
Node(
SYMBOL.suite,
[
Newline(),
Leaf(TOKEN.INDENT, indent.value + " "),
Node(
SYMBOL.simple_stmt,
[
Node(
SYMBOL.return_stmt,
[
Name("return"),
Node(
SYMBOL.power,
Attr(
Name("self"),
Name(new_name),
),
prefix=" ",
),
],
),
Newline(),
],
),
Leaf(TOKEN.DEDENT, "\n" + indent),
],
),
],
prefix=indent,
),
],
)
setter = Node(
SYMBOL.decorated,
[
Node(
SYMBOL.decorator,
[
Leaf(TOKEN.AT, "@"),
Node(
SYMBOL.dotted_name,
[Name(old_name), Dot(), Name("setter")],
),
Leaf(TOKEN.NEWLINE, "\n"),
],
),
Node(
SYMBOL.funcdef,
[
Name("def", indent),
Name(old_name, prefix=" "),
Node(
SYMBOL.parameters,
[
LParen(),
Node(
SYMBOL.typedargslist,
[
Name("self"),
Comma(),
Name("value", prefix=" "),
],
),
RParen(),
],
),
Leaf(TOKEN.COLON, ":"),
Node(
SYMBOL.suite,
[
Newline(),
Leaf(TOKEN.INDENT, indent + " "),
Node(
SYMBOL.simple_stmt,
[
Node(
SYMBOL.expr_stmt,
[
Node(
SYMBOL.power,
Attr(
Name("self"),
Name(new_name),
),
),
Leaf(
TOKEN.EQUAL,
"=",
prefix=" ",
),
Name("value", prefix=" "),
],
),
Newline(),
],
),
Leaf(TOKEN.DEDENT, "\n" + indent),
],
),
],
prefix=indent,
),
],
)
suite.insert_child(-1, getter)
suite.insert_child(-1, setter)
prev = find_previous(getter, TOKEN.DEDENT, recursive=True)
curr = find_last(setter, TOKEN.DEDENT, recursive=True)
assert isinstance(prev, Leaf) and isinstance(curr, Leaf)
prev.prefix, curr.prefix = curr.prefix, prev.prefix
prev.value, curr.value = curr.value, prev.value
def gdaltest_fail_reason_to_assert(node, capture, filename):
"""
Converts an entire if statement into an assertion.
if x == y:
print("a")
gdal.post_reason('b')
print("c")
return 'fail'
-->
assert x != y, 'foo'
Uses post_reason(x) or print(x) as valid reasons.
Prefers post_reason(x).
Ignores reasons that are sub-expressions of the condition
(ie, will ignore `x` as a reason in the above example.)
Attempts to strip out low-value print/post_reason calls.
If they can't all be stripped out, falls back to using an if statement with pytest.fail():
if x == y:
print("a", a)
pytest.fail("b")
"""
if flags["debug"]:
print(f"expression: {capture}")
returntype = string_value(capture["returntype"])
if returntype != "fail":
# only handle fails for now, tackle others later
return
condition = capture["condition"]
condition_leaves = {
n.value
for n in condition.leaves() if n.type in (TOKEN.NAME, TOKEN.STRING)
}
candidates = []
# Find a reason. Prefer post_reason reasons rather than print statements.
for stmt in capture['reason_candidates']:
power = stmt.children[0]
if power.children[0].value == 'print':
candidates.append(power.children[1].children[1])
for stmt in capture['reason_candidates']:
power = stmt.children[0]
if power.children[0].value == 'gdaltest':
candidates.append(power.children[2].children[1])
elif power.children[0].value == 'post_reason':
candidates.append(power.children[1].children[1])
candidates.reverse()
IGNORE_REASON_STRING = re.compile(
r'(got %.* expected .*)|(.* returned %.* instead of.*)', re.I)
reason = None
for i, candidate in reversed(list(enumerate(candidates[:]))):
# Don't include reasons that are actually expressions used in the comparison itself.
# These are already printed by pytest in the event of the assertion failing
candidate_leaves = {
n.value
for n in candidate.leaves() if n.type in (TOKEN.NAME, TOKEN.STRING)
}
if candidate_leaves.issubset(condition_leaves):
# Reason is just made up of expressions already used in the comparison; remove it
safe_remove_from_suite(candidate.parent.parent.parent)
candidates.pop(i)
continue
if any(leaf.type == TOKEN.STRING
and IGNORE_REASON_STRING.match(string_value(leaf))
for leaf in candidate.leaves()):
# looks kind of useless too; pytest will output the got/expected values.
safe_remove_from_suite(candidate.parent.parent.parent)
candidates.pop(i)
continue
# Keep this reason.
reason = candidate
if reason:
# remove the winning reason node from the tree
safe_remove_from_suite(reason.parent.parent.parent)
reason.remove()
candidates.remove(reason)
if not candidates:
# all print/post_reason calls were removed.
# So we can convert the if statement to an assert.
if reason:
reason = parenthesize_if_not_already(reason.clone())
assertion = Assert(
[parenthesize_if_multiline(invert_condition(condition))],
reason,
prefix=node.prefix,
)
if flags["debug"]:
print(f"Replacing:\n\t{node}")
print(f"With: {assertion}")
print()
replace_if_with_assert(node, capture['dedent'], assertion)
else:
# At least one print statement remains.
# We need to keep the if statement, and use a `pytest.fail(reason)`.
replacement = Attr(
kw("pytest", prefix=capture["return_call"].prefix),
kw(returntype, prefix=""),
) + [ArgList(listify(reason))]
# Adds a 'import pytest' if there wasn't one already
touch_import(None, "pytest", node)
capture["return_call"].replace(replacement)