def test_check_by_module(self): simplefs(self.root, SIMPLE_FOO) with add_to_pypath(self.root): retcode, lines, _ = call_check(["foo"]) self.assertEqual(retcode, 1) self.assertEqual(len(lines), 1) self.assertIn("foo.py:3: error: false when calling foofn", lines[0])
def test_added_file(tmp_path: Path): simplefs(tmp_path, CORRECT_FOO) watcher = Watcher([tmp_path]) assert watcher.check_changed() assert not watcher.check_changed() simplefs(tmp_path, EMPTY_BAR) assert watcher.check_changed()
def test_check_by_class(self): simplefs(self.root, FOO_CLASS) with add_to_pypath(self.root): retcode, lines, _ = call_check(["foo.Fooey"]) self.assertEqual(retcode, 1) self.assertEqual(len(lines), 1) self.assertIn("foo.py:4: error: false when calling incr", lines[0])
def test_check_failure_via_main(self): simplefs(self.root, SIMPLE_FOO) try: sys.stdout = io.StringIO() self.assertEqual(1, unwalled_main(["check", str(self.root / "foo.py")])) finally: sys.stdout = sys.__stdout__
def test_check_nonexistent_filename(self): simplefs(self.root, SIMPLE_FOO) retcode, _, errlines = call_check([str(self.root / "notexisting.py")]) self.assertEqual(retcode, 2) self.assertEqual(len(errlines), 1) self.assertIn("File not found", errlines[0]) self.assertIn("notexisting.py", errlines[0])
def test_check_nonexistent_module(self): simplefs(self.root, SIMPLE_FOO) retcode, _, errlines = call_check(["notexisting"]) self.assertEqual(retcode, 2) self.assertEqual( "ModuleNotFoundError: No module named 'notexisting'", errlines[-1] )
def test_removed_file(tmp_path: Path): simplefs(tmp_path, CORRECT_FOO) simplefs(tmp_path, EMPTY_BAR) watcher = Watcher([tmp_path]) assert watcher.check_changed() assert not watcher.check_changed() (tmp_path / "bar.py").unlink() assert watcher.check_changed()
def test_watch(self): # Just to make sure nothing explodes simplefs(self.root, SIMPLE_FOO) retcode = watch( Namespace(directory=[str(self.root)]), AnalysisOptionSet(), max_watch_iterations=2, ) self.assertEqual(retcode, 0)
def test_directives(self): simplefs(self.root, DIRECTIVES_TREE) ret, out, err = call_check( [str(self.root)], AnalysisOptionSet(analysis_kind=[AnalysisKind.asserts]) ) self.assertEqual(err, []) self.assertEqual(ret, 1) self.assertRegex(out[0], r"innermod.py:5: error: Exception: for any input") self.assertEqual(len(out), 1)
def test_modified_file(tmp_path: Path): simplefs(tmp_path, CORRECT_FOO) watcher = Watcher([tmp_path]) assert watcher.check_changed() assert not watcher.check_changed() time.sleep(0.01) # Ensure mtime is actually different! simplefs(tmp_path, BUGGY_FOO) assert watcher.check_changed() assert not watcher.check_changed()
def test_directives_on_check_with_linenumbers(self): simplefs(self.root, DIRECTIVES_TREE) ret, out, err = call_check( [str(self.root / "outerpkg" / "innerpkg" / "innermod.py") + ":5"], AnalysisOptionSet(analysis_kind=[AnalysisKind.asserts]), ) self.assertEqual(err, []) self.assertEqual(ret, 1) self.assertRegex(out[0], r"innermod.py:5: error: Exception: for any input") self.assertEqual(len(out), 1)
def test_diff_behavior_same(self): simplefs(self.root, SIMPLE_FOO) with add_to_pypath(self.root): retcode, lines = call_diffbehavior("foo.foofn", str(self.root / "foo.py:2")) self.assertEqual( lines, [ "No differences found. (attempted 2 iterations)", "All paths exhausted, functions are likely the same!", ], ) self.assertEqual(retcode, 0)
def test_main_as_subprocess(tmp_path: Path): # This helps check things like addaudithook() which we don't want to run inside # the testing process. simplefs(tmp_path, SIMPLE_FOO) completion = subprocess.run( ["python", "-m", "crosshair", "check", str(tmp_path)], capture_output=True, text=True, ) assert completion.returncode == 1 assert "foo.py:3: error: false when calling foofn" in completion.stdout assert completion.stderr == ""
def test_diff_behavior_via_main(self): simplefs(self.root, SIMPLE_FOO) sys.stdout = io.StringIO() try: with add_to_pypath(self.root): self.assertEqual( 0, unwalled_main(["diffbehavior", "foo.foofn", "foo.foofn"]) ) finally: out = sys.stdout.getvalue() sys.stdout = sys.__stdout__ self.assertRegex(out, "No differences found")
def test_report_confirmation(self): simplefs(self.root, FOO_WITH_CONFIRMABLE_AND_PRE_UNSAT) retcode, lines, _ = call_check([str(self.root / "foo.py")]) self.assertEqual(retcode, 0) self.assertEqual(lines, []) # Now, turn on confirmations with the `--report_all` option: retcode, lines, _ = call_check( [str(self.root / "foo.py")], options=AnalysisOptionSet(report_all=True) ) self.assertEqual(retcode, 0) self.assertEqual(len(lines), 2) output_text = "\n".join(lines) self.assertIn("foo.py:3: info: Confirmed over all paths.", output_text) self.assertIn("foo.py:7: info: Unable to meet precondition.", output_text)
def DISABLE_TODO_test_assert_mode_e2e(self): simplefs(self.root, ASSERT_BASED_FOO) try: sys.stdout = io.StringIO() exitcode = unwalled_main( ["check", self.root / "foo.py", "--analysis_kind=asserts"] ) finally: out = sys.stdout.getvalue() sys.stdout = sys.__stdout__ self.assertEqual(exitcode, 1) self.assertRegex( out, r"foo.py\:8\: error\: AssertionError\: when calling foofn\(x \= 100\)" ) self.assertEqual(len([l for l in out.split("\n") if l]), 1)
def test_check_ok_via_main(self): # contract is assert-based, but we do not analyze that type. simplefs(self.root, ASSERT_BASED_FOO) try: sys.stdout = io.StringIO() exitcode = unwalled_main( [ "check", str(self.root / "foo.py"), "--analysis_kind=PEP316,icontract", ] ) self.assertEqual(exitcode, 0) finally: sys.stdout = sys.__stdout__
def test_diff_behavior_different(self): simplefs( self.root, { "foo.py": """ def add(x: int, y: int) -> int: return x + y def faultyadd(x: int, y: int) -> int: return 42 if (x, y) == (10, 10) else x + y """ }, ) with add_to_pypath(self.root): retcode, lines = call_diffbehavior("foo.add", "foo.faultyadd") self.assertEqual(retcode, 1) self.assertEqual( lines, [ "Given: (x=10, y=10),", " foo.add : returns 20", " foo.faultyadd : returns 42", ], )
def test_diff_behavior_targeting_error(self): simplefs(self.root, SIMPLE_FOO) with add_to_pypath(self.root): retcode, lines = call_diffbehavior("foo.foofn", "foo") self.assertEqual(retcode, 2) self.assertEqual(lines, ['"foo" does not target a function.'])
def test_check_circular_with_guard(self): simplefs(self.root, CIRCULAR_WITH_GUARD) with add_to_pypath(self.root): retcode, lines, _ = call_check([str(self.root / "first.py")]) self.assertEqual(retcode, 0)
def test_load_file(self): simplefs(self.root, SIMPLE_FOO) module = load_file(join(self.root, "foo.py")) self.assertNotEqual(module, None) self.assertEqual(module.foofn(5), 6)
def test_check_by_filename(self): simplefs(self.root, SIMPLE_FOO) retcode, lines, _ = call_check([str(self.root / "foo.py")]) self.assertEqual(retcode, 1) self.assertEqual(len(lines), 1) self.assertIn("foo.py:3: error: false when calling foofn", lines[0])
def test_cover(tmp_path: Path, capsys: pytest.CaptureFixture[str]): simplefs(tmp_path, SIMPLE_FOO) with add_to_pypath(str(tmp_path)): assert unwalled_main(["cover", "foo.foofn"]) == 0 assert capsys.readouterr().out == "foofn(0)\n"
def test_check_by_package(self): simplefs(self.root, OUTER_INNER) with add_to_pypath(self.root): retcode, lines, _ = call_check(["outer.inner.foofn"]) self.assertEqual(retcode, 0) self.assertEqual(len(lines), 0)
def test_package_directives(tmp_path: Path): simplefs(tmp_path, DIRECTIVES_TREE) with add_to_pypath(tmp_path): innermod = importlib.import_module("pkg1.pkg2.pkg3.mod") assert collect_options(innermod) == AnalysisOptionSet( enabled=True, max_iterations=5, per_condition_timeout=42)
def test_check_nonexistent_member(self): simplefs(self.root, OUTER_INNER) with add_to_pypath(self.root): self.assertRaises(NotFound, lambda: call_check(["outer.inner.nonexistent"]))