def test_example_coverage(self) -> None:
# Try to get examples that highlist the differences in the code.
# Here, we add more conditions for the `return True` path and
# another case where we used to just `return False`.
def isack1(s: str) -> bool:
if s in ("y", "yes"):
return True
return False
def isack2(s: str) -> Optional[bool]:
if s in ("y", "yes", "Y", "YES"):
return True
if s in ("n", "no", "N", "NO"):
return False
return None
diffs = diff_behavior(
FunctionInfo.from_fn(isack1),
FunctionInfo.from_fn(isack2),
DEFAULT_OPTIONS.overlay(max_iterations=20,
per_condition_timeout=5),
)
debug("diffs=", diffs)
assert not isinstance(diffs, str)
return_vals = set(
(d.result1.return_repr, d.result2.return_repr) for d in diffs)
self.assertEqual(return_vals, {("False", "None"), ("False", "True")})
def test_icontract_class(self):
messages = run_checkables(
analyze_class(
IcontractB,
DEFAULT_OPTIONS.overlay(
analysis_kind=[AnalysisKind.icontract]),
))
messages = {(m.state, m.line, m.message)
for m in messages if m.state != MessageType.CONFIRMED}
self.assertEqual(
messages,
{
(
MessageType.POST_FAIL,
114,
'"@icontract.invariant(lambda self: self.x > 0)" yields false '
"when calling break_parent_invariant(self = instance of B(10))",
),
(
MessageType.POST_FAIL,
117,
'"@icontract.invariant(lambda self: self.x < 100)" yields false '
"when calling break_my_invariant(self = instance of B(10))",
),
},
)
def test_diff_staticmethod(self):
diffs = diff_behavior(
walk_qualname(Base, "staticfoo"),
foo2,
DEFAULT_OPTIONS.overlay(max_iterations=10),
)
self.assertEqual(diffs, [])
def check(args: argparse.Namespace, options: AnalysisOptionSet, stdout: TextIO,
stderr: TextIO) -> int:
any_problems = False
try:
entities = list(load_files_or_qualnames(args.target))
except FileNotFoundError as exc:
print(f'File not found: "{exc.args[0]}"', file=stderr)
return 2
except ErrorDuringImport as exc:
cause = exc.__cause__ if exc.__cause__ is not None else exc
print(f"Could not import your code:\n", file=stderr)
traceback.print_exception(type(cause),
cause,
cause.__traceback__,
file=stderr)
return 2
full_options = DEFAULT_OPTIONS.overlay(
report_verbose=False).overlay(options)
for entity in entities:
debug("Check ", getattr(entity, "__name__", str(entity)))
for message in run_checkables(analyze_any(entity, options)):
line = describe_message(message, full_options)
if line is None:
continue
stdout.write(line + "\n")
debug("Traceback for output message:\n", message.traceback)
if message.state > MessageType.PRE_UNSAT:
any_problems = True
return 1 if any_problems else 0
def test_AnalysisOptions_split_limits() -> None:
options = DEFAULT_OPTIONS.overlay(per_path_timeout=10.0, max_iterations=16)
part1, part2 = options.split_limits(0.1)
assert part1.per_path_timeout == 1.0
assert part2.per_path_timeout == 9.0
assert part1.max_iterations == 2
assert part2.max_iterations == 14
def unwalled_main(cmd_args: Union[List[str], argparse.Namespace]) -> None:
if isinstance(cmd_args, argparse.Namespace):
args = cmd_args
else:
args = command_line_parser().parse_args(cmd_args)
set_debug(args.verbose)
options = option_set_from_dict(args.__dict__)
if sys.path and sys.path[0] != "":
# fall back to current directory to look up modules
sys.path.append("")
if args.action == "check":
exitcode = check(args, options, sys.stdout, sys.stderr)
elif args.action == "diffbehavior":
defaults = DEFAULT_OPTIONS.overlay(
AnalysisOptionSet(
per_condition_timeout=2.5,
per_path_timeout=30.0, # mostly, we don't want to time out paths
))
exitcode = diffbehavior(args, defaults.overlay(options), sys.stdout,
sys.stderr)
elif args.action == "watch":
exitcode = watch(args, options)
else:
print(f'Unknown action: "{args.action}"', file=sys.stderr)
exitcode = 2
sys.exit(exitcode)
def test_icontract_snapshots(self):
messages = analyze_function(
icontract_appender,
DEFAULT_OPTIONS.overlay(
analysis_kind=[AnalysisKind.icontract]),
)
self.assertEqual(*check_messages(
messages, state=MessageType.POST_FAIL, line=95, column=0))
def test_diff_behavior_different(self) -> None:
diffs = diff_behavior(foo1, foo3,
DEFAULT_OPTIONS.overlay(max_iterations=10))
self.assertEqual(len(diffs), 1)
diff = diffs[0]
assert isinstance(diff, BehaviorDiff)
self.assertGreater(int(diff.args["x"]), 1000)
self.assertEqual(diff.result1.return_repr, "100")
self.assertEqual(diff.result2.return_repr, "1000")
def test_icontract_snapshots(self):
messages = analyze_function(
icontract_appender,
DEFAULT_OPTIONS.overlay(analysis_kind=[AnalysisKind.icontract]),
)
line = icontract_appender.__wrapped__.__code__.co_firstlineno + 1
self.assertEqual(
*check_messages(
messages, state=MessageType.POST_FAIL, line=line, column=0
)
)
def test_asserts(self):
messages = analyze_function(
remove_smallest_with_asserts,
DEFAULT_OPTIONS.overlay(
analysis_kind=[AnalysisKind.asserts],
max_iterations=10,
per_condition_timeout=5,
),
)
self.assertEqual(*check_messages(
messages, state=MessageType.EXEC_ERR, line=85, column=0))
def test_diff_method(self):
diffs = diff_behavior(
walk_qualname(Base, "foo"),
walk_qualname(Derived, "foo"),
DEFAULT_OPTIONS.overlay(max_iterations=10),
)
assert isinstance(diffs, list)
self.assertEqual(
[(d.result1.return_repr, d.result2.return_repr) for d in diffs],
[("10", "11")],
)
def unwalled_main(cmd_args: Union[List[str], argparse.Namespace]) -> int:
parser = command_line_parser()
if isinstance(cmd_args, argparse.Namespace):
args = cmd_args
else:
args = parser.parse_args(cmd_args)
if not args.action:
parser.print_help(sys.stderr)
return 2
set_debug(args.verbose)
debug("Installed plugins:", installed_plugins)
options = option_set_from_dict(args.__dict__)
# fall back to current directory to look up modules
with add_to_pypath(*([""] if sys.path and sys.path[0] != "" else [])):
if args.action == "check":
return check(args, options, sys.stdout, sys.stderr)
elif args.action == "diffbehavior":
defaults = DEFAULT_OPTIONS.overlay(
AnalysisOptionSet(
per_condition_timeout=2.5,
per_path_timeout=
30.0, # mostly, we don't want to time out paths
))
return diffbehavior(args, defaults.overlay(options), sys.stdout,
sys.stderr)
elif args.action == "cover":
defaults = DEFAULT_OPTIONS.overlay(
AnalysisOptionSet(
per_condition_timeout=2.5,
per_path_timeout=
30.0, # mostly, we don't want to time out paths
))
return cover(args, defaults.overlay(options), sys.stdout,
sys.stderr)
elif args.action == "watch":
return watch(args, options)
else:
print(f'Unknown action: "{args.action}"', file=sys.stderr)
return 2
def run_watch_loop(watcher, max_watch_iterations=sys.maxsize) -> None:
restart = True
stats: Counter[str] = Counter()
active_messages: Dict[Tuple[str, int], AnalysisMessage]
for itr_num in range(max_watch_iterations):
if restart:
clear_screen()
clear_line("-")
line = f" Analyzing {len(watcher._modtimes)} files."
sys.stdout.write(color(line, AnsiColor.OKBLUE))
sys.stdout.flush()
max_condition_timeout = 0.5
restart = False
stats = Counter()
active_messages = {}
else:
time.sleep(0.5)
max_condition_timeout *= 2
for curstats, messages in watcher.run_iteration(max_condition_timeout):
debug("stats", curstats, messages)
stats.update(curstats)
clear_screen()
if messages_merged(active_messages, messages):
linecache.checkcache()
options = DEFAULT_OPTIONS.overlay(watcher._options)
for message in active_messages.values():
lines = long_describe_message(message, options)
if lines is None:
continue
clear_line("-")
print(lines, end="")
clear_line("-")
num_files = len(watcher._modtimes)
if len(watcher._paths) > 1:
loc_desc = f"{num_files} files"
else:
path_desc = Path(next(iter(watcher._paths))).parts[-1]
if num_files > 1:
loc_desc = f'"{path_desc}" ({num_files} files)'
else:
loc_desc = f'"{path_desc}"'
line = f' Analyzed {stats["num_paths"]} paths in {loc_desc}.'
sys.stdout.write(color(line, AnsiColor.OKBLUE))
sys.stdout.flush()
if watcher._change_flag:
watcher._change_flag = False
restart = True
line = f" Restarting analysis over {len(watcher._modtimes)} files."
sys.stdout.write(color(line, AnsiColor.OKBLUE))
sys.stdout.flush()
def test_hypothesis_counterexample_text():
messages = analyze_function(
foo,
DEFAULT_OPTIONS.overlay(
analysis_kind=[AnalysisKind.hypothesis],
max_iterations=10,
per_condition_timeout=20,
per_path_timeout=5,
),
)
actual, expected = check_messages(
messages,
state=MessageType.EXEC_ERR,
message="AssertionError: assert False when calling foo(x = False)",
)
assert actual == expected
def test_diff_behavior_mutation(self) -> None:
def cut_out_item1(a: List[int], i: int):
a[i:i + 1] = []
def cut_out_item2(a: List[int], i: int):
a[:] = a[:i] + a[i + 1:]
# TODO: this takes longer than I'd like (few iterations though):
opts = DEFAULT_OPTIONS.overlay(max_iterations=20,
per_path_timeout=10,
per_condition_timeout=10)
diffs = diff_behavior(
FunctionInfo.from_fn(cut_out_item1),
FunctionInfo.from_fn(cut_out_item2),
opts,
)
assert not isinstance(diffs, str)
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertGreater(len(diff.args["a"]), 1)
self.assertEqual(diff.args["i"], "-1")
def run_watch_loop(self, max_watch_iterations=sys.maxsize) -> None:
restart = True
stats: Counter[str] = Counter()
active_messages: Dict[Tuple[str, int], AnalysisMessage]
for itr_num in range(max_watch_iterations):
if restart:
clear_screen()
clear_line("-")
line = f" Analyzing {len(self._modtimes)} files. \r"
sys.stdout.write(color(line, AnsiColor.OKBLUE))
max_condition_timeout = 0.5
restart = False
stats = Counter()
active_messages = {}
else:
time.sleep(0.5)
max_condition_timeout *= 2
for curstats, messages in self.run_iteration(
max_condition_timeout):
debug("stats", curstats, messages)
stats.update(curstats)
if messages_merged(active_messages, messages):
linecache.checkcache()
clear_screen()
options = DEFAULT_OPTIONS.overlay(self._options)
for message in active_messages.values():
lines = long_describe_message(message, options)
if lines is None:
continue
clear_line("-")
print(lines, end="")
clear_line("-")
line = f' Analyzed {stats["num_paths"]} paths in {len(self._modtimes)} files. \r'
sys.stdout.write(color(line, AnsiColor.OKBLUE))
if self._change_flag:
self._change_flag = False
restart = True
line = f" Restarting analysis over {len(self._modtimes)} files. \r"
sys.stdout.write(color(line, AnsiColor.OKBLUE))
from crosshair.path_cover import CoverageType
from crosshair.core_and_libs import *
def _foo(x: int) -> int:
if x > 100:
return 100
return x
def _regex(x: str) -> bool:
compiled = re.compile("f(o)+")
return bool(compiled.fullmatch(x))
OPTS = DEFAULT_OPTIONS.overlay(max_iterations=10, per_condition_timeout=10.0)
foo = FunctionInfo.from_fn(_foo)
regex = FunctionInfo.from_fn(_regex)
def test_path_cover() -> None:
paths = list(path_cover(foo, OPTS, CoverageType.OPCODE))
assert len(paths) == 2
small, large = sorted(paths, key=lambda p: p.result) # type: ignore
assert large.result == 100
assert large.args.arguments["x"] > 100
assert small.result == small.args.arguments["x"]
def test_path_cover_regex() -> None:
paths = list(path_cover(regex, OPTS, CoverageType.OPCODE))
def test_diff_behavior_same(self) -> None:
diffs = diff_behavior(foo1, foo2,
DEFAULT_OPTIONS.overlay(max_iterations=10))
self.assertEqual(diffs, [])
def run_watch_loop(
watcher: Watcher,
max_watch_iterations: int = sys.maxsize,
term_lines_rewritable: bool = True,
) -> None:
restart = True
stats: Counter[str] = Counter()
active_messages: Dict[Tuple[str, int], AnalysisMessage]
for _ in range(max_watch_iterations):
if restart:
clear_screen()
print_divider("-")
line = f" Analyzing {len(watcher._modtimes)} files."
print(color(line, AnsiColor.OKBLUE), end="")
max_condition_timeout = 0.5
restart = False
stats = Counter()
active_messages = {}
else:
time.sleep(0.1)
max_condition_timeout *= 2
for curstats, messages in watcher.run_iteration(max_condition_timeout):
messages = [m for m in messages if m.state > MessageType.PRE_UNSAT]
stats.update(curstats)
if messages_merged(active_messages, messages):
linecache.checkcache()
clear_screen()
options = DEFAULT_OPTIONS.overlay(watcher._options)
for message in active_messages.values():
lines = long_describe_message(message, options)
if lines is None:
continue
print_divider("-")
print(lines, end="")
print_divider("-")
else:
if term_lines_rewritable:
print("\r", end="")
else:
print(".", end="")
continue
num_files = len(watcher._modtimes)
if len(watcher._paths) > 1:
loc_desc = f"{num_files} files"
else:
path_parts = Path(next(iter(watcher._paths))).parts
path_desc = path_parts[-1] if path_parts else "."
if num_files > 1:
loc_desc = f'"{path_desc}" ({num_files} files)'
else:
loc_desc = f'"{path_desc}"'
if term_lines_rewritable:
line = f' Analyzed {stats["num_paths"]} paths in {loc_desc}. '
else:
line = f" Analyzing paths in {loc_desc}: "
print(color(line, AnsiColor.OKBLUE), end="")
if watcher._change_flag:
watcher._change_flag = False
restart = True
line = f" Restarting analysis over {len(watcher._modtimes)} files."
print(color(line, AnsiColor.OKBLUE), end="")
