def test_example_coverage(self) -> None: # Try to get examples that highlist the differences in the code. # Here, we add more conditions for the `return True` path and # another case where we used to just `return False`. def isack1(s: str) -> bool: if s in ("y", "yes"): return True return False def isack2(s: str) -> Optional[bool]: if s in ("y", "yes", "Y", "YES"): return True if s in ("n", "no", "N", "NO"): return False return None diffs = diff_behavior( FunctionInfo.from_fn(isack1), FunctionInfo.from_fn(isack2), DEFAULT_OPTIONS.overlay(max_iterations=20, per_condition_timeout=5), ) debug("diffs=", diffs) assert not isinstance(diffs, str) return_vals = set( (d.result1.return_repr, d.result2.return_repr) for d in diffs) self.assertEqual(return_vals, {("False", "None"), ("False", "True")})
def test_icontract_class(self): messages = run_checkables( analyze_class( IcontractB, DEFAULT_OPTIONS.overlay( analysis_kind=[AnalysisKind.icontract]), )) messages = {(m.state, m.line, m.message) for m in messages if m.state != MessageType.CONFIRMED} self.assertEqual( messages, { ( MessageType.POST_FAIL, 114, '"@icontract.invariant(lambda self: self.x > 0)" yields false ' "when calling break_parent_invariant(self = instance of B(10))", ), ( MessageType.POST_FAIL, 117, '"@icontract.invariant(lambda self: self.x < 100)" yields false ' "when calling break_my_invariant(self = instance of B(10))", ), }, )
def test_diff_staticmethod(self): diffs = diff_behavior( walk_qualname(Base, "staticfoo"), foo2, DEFAULT_OPTIONS.overlay(max_iterations=10), ) self.assertEqual(diffs, [])
def check(args: argparse.Namespace, options: AnalysisOptionSet, stdout: TextIO, stderr: TextIO) -> int: any_problems = False try: entities = list(load_files_or_qualnames(args.target)) except FileNotFoundError as exc: print(f'File not found: "{exc.args[0]}"', file=stderr) return 2 except ErrorDuringImport as exc: cause = exc.__cause__ if exc.__cause__ is not None else exc print(f"Could not import your code:\n", file=stderr) traceback.print_exception(type(cause), cause, cause.__traceback__, file=stderr) return 2 full_options = DEFAULT_OPTIONS.overlay( report_verbose=False).overlay(options) for entity in entities: debug("Check ", getattr(entity, "__name__", str(entity))) for message in run_checkables(analyze_any(entity, options)): line = describe_message(message, full_options) if line is None: continue stdout.write(line + "\n") debug("Traceback for output message:\n", message.traceback) if message.state > MessageType.PRE_UNSAT: any_problems = True return 1 if any_problems else 0
def test_AnalysisOptions_split_limits() -> None: options = DEFAULT_OPTIONS.overlay(per_path_timeout=10.0, max_iterations=16) part1, part2 = options.split_limits(0.1) assert part1.per_path_timeout == 1.0 assert part2.per_path_timeout == 9.0 assert part1.max_iterations == 2 assert part2.max_iterations == 14
def unwalled_main(cmd_args: Union[List[str], argparse.Namespace]) -> None: if isinstance(cmd_args, argparse.Namespace): args = cmd_args else: args = command_line_parser().parse_args(cmd_args) set_debug(args.verbose) options = option_set_from_dict(args.__dict__) if sys.path and sys.path[0] != "": # fall back to current directory to look up modules sys.path.append("") if args.action == "check": exitcode = check(args, options, sys.stdout, sys.stderr) elif args.action == "diffbehavior": defaults = DEFAULT_OPTIONS.overlay( AnalysisOptionSet( per_condition_timeout=2.5, per_path_timeout=30.0, # mostly, we don't want to time out paths )) exitcode = diffbehavior(args, defaults.overlay(options), sys.stdout, sys.stderr) elif args.action == "watch": exitcode = watch(args, options) else: print(f'Unknown action: "{args.action}"', file=sys.stderr) exitcode = 2 sys.exit(exitcode)
def test_icontract_snapshots(self): messages = analyze_function( icontract_appender, DEFAULT_OPTIONS.overlay( analysis_kind=[AnalysisKind.icontract]), ) self.assertEqual(*check_messages( messages, state=MessageType.POST_FAIL, line=95, column=0))
def test_diff_behavior_different(self) -> None: diffs = diff_behavior(foo1, foo3, DEFAULT_OPTIONS.overlay(max_iterations=10)) self.assertEqual(len(diffs), 1) diff = diffs[0] assert isinstance(diff, BehaviorDiff) self.assertGreater(int(diff.args["x"]), 1000) self.assertEqual(diff.result1.return_repr, "100") self.assertEqual(diff.result2.return_repr, "1000")
def test_icontract_snapshots(self): messages = analyze_function( icontract_appender, DEFAULT_OPTIONS.overlay(analysis_kind=[AnalysisKind.icontract]), ) line = icontract_appender.__wrapped__.__code__.co_firstlineno + 1 self.assertEqual( *check_messages( messages, state=MessageType.POST_FAIL, line=line, column=0 ) )
def test_asserts(self): messages = analyze_function( remove_smallest_with_asserts, DEFAULT_OPTIONS.overlay( analysis_kind=[AnalysisKind.asserts], max_iterations=10, per_condition_timeout=5, ), ) self.assertEqual(*check_messages( messages, state=MessageType.EXEC_ERR, line=85, column=0))
def test_diff_method(self): diffs = diff_behavior( walk_qualname(Base, "foo"), walk_qualname(Derived, "foo"), DEFAULT_OPTIONS.overlay(max_iterations=10), ) assert isinstance(diffs, list) self.assertEqual( [(d.result1.return_repr, d.result2.return_repr) for d in diffs], [("10", "11")], )
def unwalled_main(cmd_args: Union[List[str], argparse.Namespace]) -> int: parser = command_line_parser() if isinstance(cmd_args, argparse.Namespace): args = cmd_args else: args = parser.parse_args(cmd_args) if not args.action: parser.print_help(sys.stderr) return 2 set_debug(args.verbose) debug("Installed plugins:", installed_plugins) options = option_set_from_dict(args.__dict__) # fall back to current directory to look up modules with add_to_pypath(*([""] if sys.path and sys.path[0] != "" else [])): if args.action == "check": return check(args, options, sys.stdout, sys.stderr) elif args.action == "diffbehavior": defaults = DEFAULT_OPTIONS.overlay( AnalysisOptionSet( per_condition_timeout=2.5, per_path_timeout= 30.0, # mostly, we don't want to time out paths )) return diffbehavior(args, defaults.overlay(options), sys.stdout, sys.stderr) elif args.action == "cover": defaults = DEFAULT_OPTIONS.overlay( AnalysisOptionSet( per_condition_timeout=2.5, per_path_timeout= 30.0, # mostly, we don't want to time out paths )) return cover(args, defaults.overlay(options), sys.stdout, sys.stderr) elif args.action == "watch": return watch(args, options) else: print(f'Unknown action: "{args.action}"', file=sys.stderr) return 2
def run_watch_loop(watcher, max_watch_iterations=sys.maxsize) -> None: restart = True stats: Counter[str] = Counter() active_messages: Dict[Tuple[str, int], AnalysisMessage] for itr_num in range(max_watch_iterations): if restart: clear_screen() clear_line("-") line = f" Analyzing {len(watcher._modtimes)} files." sys.stdout.write(color(line, AnsiColor.OKBLUE)) sys.stdout.flush() max_condition_timeout = 0.5 restart = False stats = Counter() active_messages = {} else: time.sleep(0.5) max_condition_timeout *= 2 for curstats, messages in watcher.run_iteration(max_condition_timeout): debug("stats", curstats, messages) stats.update(curstats) clear_screen() if messages_merged(active_messages, messages): linecache.checkcache() options = DEFAULT_OPTIONS.overlay(watcher._options) for message in active_messages.values(): lines = long_describe_message(message, options) if lines is None: continue clear_line("-") print(lines, end="") clear_line("-") num_files = len(watcher._modtimes) if len(watcher._paths) > 1: loc_desc = f"{num_files} files" else: path_desc = Path(next(iter(watcher._paths))).parts[-1] if num_files > 1: loc_desc = f'"{path_desc}" ({num_files} files)' else: loc_desc = f'"{path_desc}"' line = f' Analyzed {stats["num_paths"]} paths in {loc_desc}.' sys.stdout.write(color(line, AnsiColor.OKBLUE)) sys.stdout.flush() if watcher._change_flag: watcher._change_flag = False restart = True line = f" Restarting analysis over {len(watcher._modtimes)} files." sys.stdout.write(color(line, AnsiColor.OKBLUE)) sys.stdout.flush()
def test_hypothesis_counterexample_text(): messages = analyze_function( foo, DEFAULT_OPTIONS.overlay( analysis_kind=[AnalysisKind.hypothesis], max_iterations=10, per_condition_timeout=20, per_path_timeout=5, ), ) actual, expected = check_messages( messages, state=MessageType.EXEC_ERR, message="AssertionError: assert False when calling foo(x = False)", ) assert actual == expected
def test_diff_behavior_mutation(self) -> None: def cut_out_item1(a: List[int], i: int): a[i:i + 1] = [] def cut_out_item2(a: List[int], i: int): a[:] = a[:i] + a[i + 1:] # TODO: this takes longer than I'd like (few iterations though): opts = DEFAULT_OPTIONS.overlay(max_iterations=20, per_path_timeout=10, per_condition_timeout=10) diffs = diff_behavior( FunctionInfo.from_fn(cut_out_item1), FunctionInfo.from_fn(cut_out_item2), opts, ) assert not isinstance(diffs, str) self.assertEqual(len(diffs), 1) diff = diffs[0] self.assertGreater(len(diff.args["a"]), 1) self.assertEqual(diff.args["i"], "-1")
def run_watch_loop(self, max_watch_iterations=sys.maxsize) -> None: restart = True stats: Counter[str] = Counter() active_messages: Dict[Tuple[str, int], AnalysisMessage] for itr_num in range(max_watch_iterations): if restart: clear_screen() clear_line("-") line = f" Analyzing {len(self._modtimes)} files. \r" sys.stdout.write(color(line, AnsiColor.OKBLUE)) max_condition_timeout = 0.5 restart = False stats = Counter() active_messages = {} else: time.sleep(0.5) max_condition_timeout *= 2 for curstats, messages in self.run_iteration( max_condition_timeout): debug("stats", curstats, messages) stats.update(curstats) if messages_merged(active_messages, messages): linecache.checkcache() clear_screen() options = DEFAULT_OPTIONS.overlay(self._options) for message in active_messages.values(): lines = long_describe_message(message, options) if lines is None: continue clear_line("-") print(lines, end="") clear_line("-") line = f' Analyzed {stats["num_paths"]} paths in {len(self._modtimes)} files. \r' sys.stdout.write(color(line, AnsiColor.OKBLUE)) if self._change_flag: self._change_flag = False restart = True line = f" Restarting analysis over {len(self._modtimes)} files. \r" sys.stdout.write(color(line, AnsiColor.OKBLUE))
from crosshair.path_cover import CoverageType from crosshair.core_and_libs import * def _foo(x: int) -> int: if x > 100: return 100 return x def _regex(x: str) -> bool: compiled = re.compile("f(o)+") return bool(compiled.fullmatch(x)) OPTS = DEFAULT_OPTIONS.overlay(max_iterations=10, per_condition_timeout=10.0) foo = FunctionInfo.from_fn(_foo) regex = FunctionInfo.from_fn(_regex) def test_path_cover() -> None: paths = list(path_cover(foo, OPTS, CoverageType.OPCODE)) assert len(paths) == 2 small, large = sorted(paths, key=lambda p: p.result) # type: ignore assert large.result == 100 assert large.args.arguments["x"] > 100 assert small.result == small.args.arguments["x"] def test_path_cover_regex() -> None: paths = list(path_cover(regex, OPTS, CoverageType.OPCODE))
def test_diff_behavior_same(self) -> None: diffs = diff_behavior(foo1, foo2, DEFAULT_OPTIONS.overlay(max_iterations=10)) self.assertEqual(diffs, [])
def run_watch_loop( watcher: Watcher, max_watch_iterations: int = sys.maxsize, term_lines_rewritable: bool = True, ) -> None: restart = True stats: Counter[str] = Counter() active_messages: Dict[Tuple[str, int], AnalysisMessage] for _ in range(max_watch_iterations): if restart: clear_screen() print_divider("-") line = f" Analyzing {len(watcher._modtimes)} files." print(color(line, AnsiColor.OKBLUE), end="") max_condition_timeout = 0.5 restart = False stats = Counter() active_messages = {} else: time.sleep(0.1) max_condition_timeout *= 2 for curstats, messages in watcher.run_iteration(max_condition_timeout): messages = [m for m in messages if m.state > MessageType.PRE_UNSAT] stats.update(curstats) if messages_merged(active_messages, messages): linecache.checkcache() clear_screen() options = DEFAULT_OPTIONS.overlay(watcher._options) for message in active_messages.values(): lines = long_describe_message(message, options) if lines is None: continue print_divider("-") print(lines, end="") print_divider("-") else: if term_lines_rewritable: print("\r", end="") else: print(".", end="") continue num_files = len(watcher._modtimes) if len(watcher._paths) > 1: loc_desc = f"{num_files} files" else: path_parts = Path(next(iter(watcher._paths))).parts path_desc = path_parts[-1] if path_parts else "." if num_files > 1: loc_desc = f'"{path_desc}" ({num_files} files)' else: loc_desc = f'"{path_desc}"' if term_lines_rewritable: line = f' Analyzed {stats["num_paths"]} paths in {loc_desc}. ' else: line = f" Analyzing paths in {loc_desc}: " print(color(line, AnsiColor.OKBLUE), end="") if watcher._change_flag: watcher._change_flag = False restart = True line = f" Restarting analysis over {len(watcher._modtimes)} files." print(color(line, AnsiColor.OKBLUE), end="")