class TestFuzz(unittest.TestCase):
@settings(suppress_health_check=[HealthCheck.too_slow])
@given(from_grammar().map(ast.parse))
def test_does_not_crash_on_any_valid_code(self, syntax_tree):
# Given any syntatically-valid source code, flake8-bugbear should
# not crash. This tests doesn't check that we do the *right* thing,
# just that we don't crash on valid-if-poorly-styled code!
BugBearVisitor(filename="<string>", lines=[]).visit(syntax_tree)
def test_does_not_crash_on_site_code(self):
# Because the generator isn't perfect, we'll also test on all the code
# we can easily find in our current Python environment - this includes
# the standard library, and all installed packages.
for base in sorted(set(site.PREFIXES)):
for dirname, _, files in os.walk(base):
for f in files:
if f.endswith(".py"):
BugBearChecker(filename=str(Path(dirname) / f))
def test_does_not_crash_on_tuple_expansion_in_except_statement(self):
# akin to test_does_not_crash_on_any_valid_code
# but targets a rare case that's not covered by hypothesmith.from_grammar
# see https://github.com/PyCQA/flake8-bugbear/issues/153
syntax_tree = ast.parse("grey_list = (ValueError,)\n"
"black_list = (TypeError,)\n"
"try:\n"
" int('1e3')\n"
"except (*grey_list, *black_list):\n"
" print('error caught')")
BugBearVisitor(filename="<string>", lines=[]).visit(syntax_tree)
class TestLib2to3(unittest.TestCase):
# TODO: https://docs.python.org/3/library/2to3.html
@unittest.expectedFailure
@example("#")
@example("\n\\\n")
@example("#\n\x0cpass#\n")
@given(source_code=hypothesmith.from_grammar().map(fixup).filter(str.strip)
)
@settings.get_profile("slow")
def test_lib2to3_tokenize_round_trip(self, source_code):
tokens = []
def token_eater(*args):
tokens.append(args)
# Round-trip invariant for full input:
# Untokenized source will match input source exactly
lib2to3_tokenize.tokenize(
io.StringIO(source_code).readline, token_eater)
full = lib2to3_tokenize.untokenize(tokens)
self.assertEqual(source_code, full)
# Round-trip invariant for limited input:
# Output text will tokenize the back to the input
part_tokens = [t[:2] for t in tokens]
partial = lib2to3_tokenize.untokenize(part_tokens)
del tokens[:]
lib2to3_tokenize.tokenize(io.StringIO(partial).readline, token_eater)
self.assertEqual(part_tokens, [t[:2] for t in tokens])
class TestTokenize(unittest.TestCase):
"""Tests that the result of `untokenize` round-trips back to the same token stream,
per https://docs.python.org/3/library/tokenize.html#tokenize.untokenize
Unfortunately these tests demonstrate that it doesn't, and thus we have
`@unittest.expectedFailure` decorators.
"""
@unittest.expectedFailure
@example("#")
@example("\n\\\n")
@example("#\n\x0cpass#\n")
@given(source_code=hypothesmith.from_grammar().map(fixup).filter(str.strip)
)
@settings.get_profile("slow")
def test_tokenize_round_trip_bytes(self, source_code):
try:
source = source_code.encode("utf-8-sig")
except UnicodeEncodeError:
reject()
tokens = list(tokenize.tokenize(io.BytesIO(source).readline))
# `outbytes` may have changed whitespace from `source`
outbytes = tokenize.untokenize(tokens)
output = list(tokenize.tokenize(io.BytesIO(outbytes).readline))
self.assertEqual(
[(tt.type, tt.string) for tt in tokens],
[(ot.type, ot.string) for ot in output],
)
# It would be nice if the round-tripped string stabilised...
# self.assertEqual(outbytes, tokenize.untokenize(output))
@unittest.expectedFailure
@example("#")
@example("\n\\\n")
@example("#\n\x0cpass#\n")
@given(source_code=hypothesmith.from_grammar().map(fixup).filter(str.strip)
)
@settings.get_profile("slow")
def test_tokenize_round_trip_string(self, source_code):
tokens = list(
tokenize.generate_tokens(io.StringIO(source_code).readline))
# `outstring` may have changed whitespace from `source_code`
outstring = tokenize.untokenize(tokens)
output = tokenize.generate_tokens(io.StringIO(outstring).readline)
self.assertEqual(
[(tt.type, tt.string) for tt in tokens],
[(ot.type, ot.string) for ot in output],
)
class TestAST(unittest.TestCase):
@unittest.skipIf(not hasattr(ast, "unparse"), reason="new in 3.9")
@given(source_code=hypothesmith.from_grammar())
@settings.get_profile("slow")
def test_ast_unparse(self, source_code):
"""Unparsing always produces code which parses back to the same ast."""
first = ast.parse(source_code)
unparsed = ast.unparse(first)
second = ast.parse(unparsed)
self.assertEqual(ast.dump(first), ast.dump(second))
class TestDis(unittest.TestCase):
@given(
code_object=hypothesmith.from_grammar().map(
lambda s: compile(s, "<string>", "exec")),
first_line=st.none() | st.integers(0, 100),
current_offset=st.none() | st.integers(0, 100),
)
@settings.get_profile("slow")
def test_disassembly(self, code_object, first_line, current_offset):
"""Exercise the dis module."""
# TODO: what other properties of this module should we test?
bcode = dis.Bytecode(code_object,
first_line=first_line,
current_offset=current_offset)
self.assertIs(code_object, bcode.codeobj)
for inst in bcode:
self.assertIsInstance(inst, dis.Instruction)
class TestFuzz(unittest.TestCase):
@given(from_grammar().map(ast.parse))
def test_does_not_crash_on_any_valid_code(self, syntax_tree):
# Given any syntatically-valid source code, flake8-bugbear should
# not crash. This tests doesn't check that we do the *right* thing,
# just that we don't crash on valid-if-poorly-styled code!
BugBearVisitor(filename="<string>", lines=[]).visit(syntax_tree)
def test_does_not_crash_on_site_code(self):
# Because the generator isn't perfect, we'll also test on all the code
# we can easily find in our current Python environment - this includes
# the standard library, and all installed packages.
for base in sorted(set(site.PREFIXES)):
for dirname, _, files in os.walk(base):
for f in files:
if f.endswith(".py"):
BugBearChecker(filename=str(Path(dirname) / f))
class FuzzTest(unittest.TestCase):
"""Fuzz-tests based on Hypothesis and Hypothesmith."""
@unittest.skipUnless(bool(os.environ.get("HYPOTHESIS", False)),
"Hypothesis not requested")
@hypothesis.given(source_code=from_grammar(start="file_input"))
def test_parsing_compilable_module_strings(self, source_code: str) -> None:
"""The `from_grammar()` strategy generates strings from Python's grammar.
This has a bunch of weird edge cases because "strings accepted by Cpython"
isn't actually the same set as "strings accepted by the grammar", and for
a few other weird reasons you can ask Zac about if you're interested.
Valid values for ``start`` are ``"single_input"``, ``"file_input"``, or
``"eval_input"``; respectively a single interactive statement, a module or
sequence of commands read from a file, and input for the eval() function.
.. warning::
DO NOT EXECUTE CODE GENERATED BY THE `from_grammar()` STRATEGY.
It could do literally anything that running Python code is able to do,
including changing, deleting, or uploading important data. Arbitrary
code can be useful, but "arbitrary code execution" can be very, very bad.
"""
self.reject_invalid_code(source_code, mode="exec")
tree = libcst.parse_module(source_code)
self.assertEqual(source_code, tree.code)
@unittest.skipUnless(bool(os.environ.get("HYPOTHESIS", False)),
"Hypothesis not requested")
@hypothesis.given(source_code=from_grammar(start="eval_input").map(
str.strip))
def test_parsing_compilable_expression_strings(self,
source_code: str) -> None:
"""Much like statements, but for expressions this time.
We change the start production of the grammar, the compile mode,
and the libCST parse function, but codegen is as for statements.
"""
self.reject_invalid_code(source_code, mode="eval")
tree = libcst.parse_expression(source_code)
self.verify_identical_asts(source_code,
libcst.Module([]).code_for_node(tree),
mode="eval")
@unittest.skipUnless(bool(os.environ.get("HYPOTHESIS", False)),
"Hypothesis not requested")
@hypothesis.given(source_code=from_grammar(
start="single_input").map(lambda s: s.replace("\n", "") + "\n"))
def test_parsing_compilable_statement_strings(self,
source_code: str) -> None:
"""Just like the above, but for statements.
We change the start production of the grammar, the compile mode,
the libCST parse function, and the codegen method.
"""
self.reject_invalid_code(source_code, mode="single")
tree = libcst.parse_statement(source_code)
self.verify_identical_asts(source_code,
libcst.Module([]).code_for_node(tree),
mode="single")
def verify_identical_asts(self, original_code: str, rendered_code: str,
mode: str) -> None:
assert mode in {"eval", "exec", "single"}
self.assertEqual(
ast.dump(ast.parse(original_code, mode=mode)),
ast.dump(ast.parse(rendered_code, mode=mode)),
)
@staticmethod
def reject_invalid_code(source_code: str, mode: str) -> None:
"""Input validation helper shared by modules, statements, and expressions."""
# We start by compiling our source code to byte code, and rejecting inputs
# where this fails. This is to guard against spurious failures due to
# e.g. `eval` only being a keyword in Python 3.7
assert mode in {"eval", "exec", "single"}
hypothesis.note(source_code)
try:
compile(source_code, "<string>", mode)
except Exception:
# We're going to check here that libCST also rejects this string.
# If libCST parses it's a test failure; if not we reject this input
# so Hypothesis spends as little time as possible exploring invalid
# code. (usually I'd use a custom mutator, but this is free so...)
parser = dict(
eval=libcst.parse_expression,
exec=libcst.parse_module,
single=libcst.parse_statement,
)
try:
tree = parser[mode](source_code)
msg = f"libCST parsed a string rejected by compile() into {tree!r}"
assert False, msg
except Exception:
hypothesis.reject()
assert False, "unreachable"
import hypothesmith
import pytest
from hypothesis import given, settings, HealthCheck
from pybetter.cli import process_file, ALL_IMPROVEMENTS
settings.register_profile(
"slow_example_generation",
suppress_health_check=HealthCheck.all(),
)
settings.load_profile("slow_example_generation")
@pytest.mark.slow
@given(generated_source=hypothesmith.from_grammar())
def test_no_crashes_on_valid_input(generated_source):
process_file(generated_source, ALL_IMPROVEMENTS)
import argparse
import hypothesmith
from hypothesis import HealthCheck, given, settings
generated_source = []
@settings(max_examples=1, suppress_health_check=[HealthCheck.return_value])
@given(hypothesmith.from_grammar(start="single_input"))
def generate(source):
print(source)
generated_source.append(source)
def main():
"""generate example scripts for consumption by other processes"""
parser = argparse.ArgumentParser()
parser.add_argument('-o', '--outfile')
args = parser.parse_args()
# @given will call the decorated function for us, meaning all values are
# already generated by the end of function call but None is returned because
# it may have been called multiple times
generate()
for some_python in generated_source:
if args.outfile:
with open(args.outfile, 'w') as out:
out.write(some_python)
else:
print(some_python)
from hypothesis import example, given, reject, strategies as st
import hypothesmith
def fixup(s):
"""Avoid known issues with tokenize() by editing the string."""
return "".join(
x for x in s if x.isprintable()).strip().strip("\\").strip() + "\n"
@pytest.mark.xfail
@example("#")
@example("\n\\\n")
@example("#\n\x0cpass#\n")
@given(source_code=hypothesmith.from_grammar().map(fixup).filter(str.strip))
def test_tokenize_round_trip_bytes(source_code):
try:
source = source_code.encode("utf-8-sig")
except UnicodeEncodeError:
reject()
tokens = list(tokenize.tokenize(io.BytesIO(source).readline))
outbytes = tokenize.untokenize(
tokens) # may have changed whitespace from source
output = list(tokenize.tokenize(io.BytesIO(outbytes).readline))
assert [(t.type, t.string) for t in tokens] == [(t.type, t.string)
for t in output]
# It would be nice if the round-tripped string stabilised. It doesn't.
# assert outbytes == tokenize.untokenize(output)
except SyntaxError:
return result
errors = [
err
for err in ComprehensionChecker(tree).run()
if not err[2].startswith(NOT_YET_FIXED)
]
assert not errors
return result
example_kwargs = {"refactor": True, "provides": frozenset(), "min_version": (3, 8)}
@given(
source_code=hypothesmith.from_grammar(auto_target=False)
| hypothesmith.from_node(auto_target=False),
refactor=st.booleans(),
provides=st.frozensets(st.from_regex(r"\A[\w\d_]+\Z").filter(str.isidentifier)),
min_version=st.sampled_from(sorted(_version_map.values())),
)
@example(source_code=TEYIT_TWO_PASS, **example_kwargs)
@example(source_code="class A:\n\x0c pass\n", **example_kwargs)
@example(
source_code="from.import(A)#",
refactor=False,
provides=frozenset(),
min_version=(3, 7),
)
# Minimum-version examples via https://github.com/jwilk/python-syntax-errors/
@example(source_code="lambda: (x := 0)\n", **example_kwargs)
FORM_FEED_CHAR = "\x0c"
# This test uses the Hypothesis and Hypothesmith libraries to generate random
# syntatically-valid Python source code and run Pycln in odd modes.
@settings(
max_examples=1750, # roughly 1750 tests/minute,
derandomize=True, # deterministic mode to avoid CI flakiness
deadline=None, # ignore Hypothesis' health checks; we already know that
suppress_health_check=HealthCheck.all(), # this is slow and filter-heavy.
)
@given(
# Note that while Hypothesmith might generate code unlike that written by
# humans, it's a general test that should pass for any *valid* source code.
# (so e.g. running it against code scraped of the internet might also help)
src_contents=hypothesmith.from_grammar()
| hypothesmith.from_node()
)
def test_idempotent_any_syntatically_valid_python(src_contents: str) -> None:
# Form feed char is detected by `pycln.utils.iou.safe_read`.
if FORM_FEED_CHAR not in src_contents:
# Before starting, let's confirm that the input string is valid Python:
compile(src_contents, "<string>", "exec") # else the bug is in hypothesmith
# Then format the code...
configs = config.Config(paths=[Path("pycln/")], all_=True)
reporter = report.Report(configs)
session_maker = refactor.Refactor(configs, reporter)
dst_contents = session_maker._code_session(src_contents)
checker = BuiltinsChecker('', 'stdin')
ret = [c for c in checker.run()]
self.assertEqual(
len(ret),
1,
)
@pytest.mark.skipif(
sys.version_info < (3, 5),
reason='This syntax is only valid in Python 3.x',
)
def test_tuple_unpacking(self):
tree = ast.parse('a, *(b, c) = 1, 2, 3')
checker = BuiltinsChecker(tree, '/home/script.py')
ret = [c for c in checker.run()]
self.assertEqual(len(ret), 0)
if sys.version_info >= (3, 6):
@given(source_code=hypothesmith.from_grammar())
def test_builtin_works_on_many_examples(source_code):
try:
source = source_code.encode('utf-8-sig')
except UnicodeEncodeError:
reject()
raise
tree = ast.parse(source)
checker = BuiltinsChecker(tree, '/home/script.py')
assert isinstance([c for c in checker.run()], list)
from blib2to3.pgen2.tokenize import TokenError
# This test uses the Hypothesis and Hypothesmith libraries to generate random
# syntatically-valid Python source code and run Black in odd modes.
@settings(
max_examples=1000, # roughly 1k tests/minute, or half that under coverage
derandomize=True, # deterministic mode to avoid CI flakiness
deadline=None, # ignore Hypothesis' health checks; we already know that
suppress_health_check=HealthCheck.all(), # this is slow and filter-heavy.
)
@given(
# Note that while Hypothesmith might generate code unlike that written by
# humans, it's a general test that should pass for any *valid* source code.
# (so e.g. running it against code scraped of the internet might also help)
src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(),
# Using randomly-varied modes helps us to exercise less common code paths.
mode=st.builds(
black.FileMode,
line_length=st.just(88) | st.integers(0, 200),
string_normalization=st.booleans(),
is_pyi=st.booleans(),
),
)
def test_idempotent_any_syntatically_valid_python(
src_contents: str, mode: black.FileMode) -> None:
# Before starting, let's confirm that the input string is valid Python:
compile(src_contents, "<string>",
"exec") # else the bug is in hypothesmith
# Then format the code...
class FuzzTest(unittest.TestCase):
"""Fuzz-tests based on Hypothesis and Hypothesmith."""
@unittest.skipUnless(bool(os.environ.get("HYPOTHESIS", False)),
"Hypothesis not requested")
@hypothesis.given(source_code=from_grammar(start="file_input"))
def test_parsing_compilable_module_strings(self, source_code: str) -> None:
"""The `from_grammar()` strategy generates strings from Python's grammar.
This has a bunch of weird edge cases because "strings accepted by Cpython"
isn't actually the same set as "strings accepted by the grammar", and for
a few other weird reasons you can ask Zac about if you're interested.
Valid values for ``start`` are ``"single_input"``, ``"file_input"``, or
``"eval_input"``; respectively a single interactive statement, a module or
sequence of commands read from a file, and input for the eval() function.
.. warning::
DO NOT EXECUTE CODE GENERATED BY THE `from_grammar()` STRATEGY.
It could do literally anything that running Python code is able to do,
including changing, deleting, or uploading important data. Arbitrary
code can be useful, but "arbitrary code execution" can be very, very bad.
"""
self.reject_invalid_code(source_code, mode="exec")
self.reject_unsupported_code(source_code)
tree = libcst.parse_module(source_code)
self.assertEqual(source_code, tree.code)
@unittest.skipUnless(bool(os.environ.get("HYPOTHESIS", False)),
"Hypothesis not requested")
@hypothesis.given(source_code=from_grammar(start="eval_input").map(
str.strip))
def test_parsing_compilable_expression_strings(self,
source_code: str) -> None:
"""Much like statements, but for expressions this time.
We change the start production of the grammar, the compile mode,
and the libCST parse function, but codegen is as for statements.
"""
self.reject_invalid_code(source_code, mode="eval")
self.reject_unsupported_code(source_code)
try:
tree = libcst.parse_expression(source_code)
self.verify_identical_asts(source_code,
libcst.Module([]).code_for_node(tree),
mode="eval")
except libcst.ParserSyntaxError:
# Unlike statements, which allow us to strip trailing whitespace,
# expressions require no whitespace or newlines. Its much more work
# to attempt to detect and strip comments and whitespace at the end
# of expressions, so instead we will reject this input. There's a
# chance we could miss some stuff here, but it should be caught by
# statement or module fuzzers. We will still catch any instance where
# expressions are parsed and rendered by LibCST in a way that changes
# the AST.
hypothesis.reject()
@unittest.skipUnless(bool(os.environ.get("HYPOTHESIS", False)),
"Hypothesis not requested")
@hypothesis.given(source_code=from_grammar(
start="single_input").map(lambda s: s.replace("\n", "") + "\n"))
def test_parsing_compilable_statement_strings(self,
source_code: str) -> None:
"""Just like the above, but for statements.
We change the start production of the grammar, the compile mode,
the libCST parse function, and the codegen method.
"""
self.reject_invalid_code(source_code, mode="single")
self.reject_unsupported_code(source_code)
tree = libcst.parse_statement(source_code)
self.verify_identical_asts(source_code,
libcst.Module([]).code_for_node(tree),
mode="single")
def verify_identical_asts(self, original_code: str, rendered_code: str,
mode: str) -> None:
assert mode in {"eval", "exec", "single"}
self.assertEqual(
ast.dump(ast.parse(original_code, mode=mode)),
ast.dump(ast.parse(rendered_code, mode=mode)),
)
@staticmethod
def reject_invalid_code(source_code: str, mode: str) -> None:
"""Input validation helper shared by modules, statements, and expressions."""
# We start by compiling our source code to byte code, and rejecting inputs
# where this fails. This is to guard against spurious failures due to
# e.g. `eval` only being a keyword in Python 3.7
assert mode in {"eval", "exec", "single"}
hypothesis.note(source_code)
try:
compile(source_code, "<string>", mode)
except Exception:
# We're going to check here that libCST also rejects this string.
# If libCST parses it's a test failure; if not we reject this input
# so Hypothesis spends as little time as possible exploring invalid
# code. (usually I'd use a custom mutator, but this is free so...)
parser = dict(
eval=libcst.parse_expression,
exec=libcst.parse_module,
single=libcst.parse_statement,
)
try:
tree = parser[mode](source_code)
msg = f"libCST parsed a string rejected by compile() into {tree!r}"
assert False, msg
except Exception:
hypothesis.reject()
assert False, "unreachable"
@staticmethod
def reject_unsupported_code(source_code: str) -> None:
"""
There are a few edge cases in Python that are too obscure and too hard to
support reasonably. If we encounter code that is generated by Hypothesis
which contains such features, we should reject it so we don't get failures
that we aren't going to fix.
"""
if "\f" in source_code:
# This is standard whitespsce, but it resets the indentation. So it
# takes the unique position of being allowed in an un-indented prefix
# while still making the program parse. We don't have a concept for
# such whitespace, so we have nowhere to put it. Consequently, while
# we can parse such code, we cannot round-trip it without losing the
# \f characters. So, since part of these fuzz tests verifies that we
# round trip perfectly, reject this.
hypothesis.reject()
settings.register_profile(
'slow',
deadline=None,
suppress_health_check=HealthCheck.all(),
)
settings.load_profile('slow')
def _fixup(string: str) -> str:
"""Avoid known issues with tokenize() by editing the string."""
return ''.join(char for char in string
if char.isprintable()).strip().strip('\\').strip() + '\n'
@given(source_code=hypothesmith.from_grammar().map(_fixup))
@settings(print_blob=True)
def test_no_exceptions(
source_code,
default_options,
parse_ast_tree,
parse_tokens,
):
"""
This testcase is a complex example of magic.
We use property based-test to generate python programs for us.
And then we ensure that our linter does not crash on arbitary input.
"""
try:
tree = parse_ast_tree(str(source_code.encode('utf-8-sig')))
"profile":
st.sampled_from(sorted(isort.settings.profiles)),
"py_version":
st.sampled_from(("auto", ) + isort.settings.VALID_PY_TARGETS),
}
kwargs = {**inferred_kwargs, **specific, **force_strategies}
return st.fixed_dictionaries({}, optional=kwargs).map(_as_config)
st.register_type_strategy(isort.Config, configs())
@hypothesis.example("import A\nimportA\r\n\n", isort.Config(), False)
@hypothesis.given(
source_code=st.lists(
from_grammar(auto_target=False)
| from_node(auto_target=False)
| from_node(libcst.Import, auto_target=False)
| from_node(libcst.ImportFrom, auto_target=False),
min_size=1,
max_size=10,
).map("\n".join),
config=st.builds(isort.Config),
disregard_skip=st.booleans(),
)
@hypothesis.seed(235738473415671197623909623354096762459)
@hypothesis.settings(suppress_health_check=[
hypothesis.HealthCheck.too_slow, hypothesis.HealthCheck.filter_too_much
])
def test_isort_is_idempotent(source_code: str, config: isort.Config,
disregard_skip: bool) -> None:
