Exemplo n.º 1
0
    def test_example_coverage(self) -> None:
        # Try to get examples that highlist the differences in the code.
        # Here, we add more conditions for the `return True` path and
        # another case where we used to just `return False`.
        def isack1(s: str) -> bool:
            if s in ("y", "yes"):
                return True
            return False

        def isack2(s: str) -> Optional[bool]:
            if s in ("y", "yes", "Y", "YES"):
                return True
            if s in ("n", "no", "N", "NO"):
                return False
            return None

        diffs = diff_behavior(
            FunctionInfo.from_fn(isack1),
            FunctionInfo.from_fn(isack2),
            DEFAULT_OPTIONS.overlay(max_iterations=20,
                                    per_condition_timeout=5),
        )
        debug("diffs=", diffs)
        assert not isinstance(diffs, str)
        return_vals = set(
            (d.result1.return_repr, d.result2.return_repr) for d in diffs)
        self.assertEqual(return_vals, {("False", "None"), ("False", "True")})
Exemplo n.º 2
0
 def test_icontract_class(self):
     messages = run_checkables(
         analyze_class(
             IcontractB,
             DEFAULT_OPTIONS.overlay(
                 analysis_kind=[AnalysisKind.icontract]),
         ))
     messages = {(m.state, m.line, m.message)
                 for m in messages if m.state != MessageType.CONFIRMED}
     self.assertEqual(
         messages,
         {
             (
                 MessageType.POST_FAIL,
                 114,
                 '"@icontract.invariant(lambda self: self.x > 0)" yields false '
                 "when calling break_parent_invariant(self = instance of B(10))",
             ),
             (
                 MessageType.POST_FAIL,
                 117,
                 '"@icontract.invariant(lambda self: self.x < 100)" yields false '
                 "when calling break_my_invariant(self = instance of B(10))",
             ),
         },
     )
Exemplo n.º 3
0
 def test_diff_staticmethod(self):
     diffs = diff_behavior(
         walk_qualname(Base, "staticfoo"),
         foo2,
         DEFAULT_OPTIONS.overlay(max_iterations=10),
     )
     self.assertEqual(diffs, [])
Exemplo n.º 4
0
def check(args: argparse.Namespace, options: AnalysisOptionSet, stdout: TextIO,
          stderr: TextIO) -> int:
    any_problems = False
    try:
        entities = list(load_files_or_qualnames(args.target))
    except FileNotFoundError as exc:
        print(f'File not found: "{exc.args[0]}"', file=stderr)
        return 2
    except ErrorDuringImport as exc:
        cause = exc.__cause__ if exc.__cause__ is not None else exc
        print(f"Could not import your code:\n", file=stderr)
        traceback.print_exception(type(cause),
                                  cause,
                                  cause.__traceback__,
                                  file=stderr)
        return 2
    full_options = DEFAULT_OPTIONS.overlay(
        report_verbose=False).overlay(options)
    for entity in entities:
        debug("Check ", getattr(entity, "__name__", str(entity)))
        for message in run_checkables(analyze_any(entity, options)):
            line = describe_message(message, full_options)
            if line is None:
                continue
            stdout.write(line + "\n")
            debug("Traceback for output message:\n", message.traceback)
            if message.state > MessageType.PRE_UNSAT:
                any_problems = True
    return 1 if any_problems else 0
def test_AnalysisOptions_split_limits() -> None:
    options = DEFAULT_OPTIONS.overlay(per_path_timeout=10.0, max_iterations=16)
    part1, part2 = options.split_limits(0.1)
    assert part1.per_path_timeout == 1.0
    assert part2.per_path_timeout == 9.0
    assert part1.max_iterations == 2
    assert part2.max_iterations == 14
Exemplo n.º 6
0
def unwalled_main(cmd_args: Union[List[str], argparse.Namespace]) -> None:
    if isinstance(cmd_args, argparse.Namespace):
        args = cmd_args
    else:
        args = command_line_parser().parse_args(cmd_args)
    set_debug(args.verbose)
    options = option_set_from_dict(args.__dict__)
    if sys.path and sys.path[0] != "":
        # fall back to current directory to look up modules
        sys.path.append("")
    if args.action == "check":
        exitcode = check(args, options, sys.stdout, sys.stderr)
    elif args.action == "diffbehavior":
        defaults = DEFAULT_OPTIONS.overlay(
            AnalysisOptionSet(
                per_condition_timeout=2.5,
                per_path_timeout=30.0,  # mostly, we don't want to time out paths
            ))
        exitcode = diffbehavior(args, defaults.overlay(options), sys.stdout,
                                sys.stderr)
    elif args.action == "watch":
        exitcode = watch(args, options)
    else:
        print(f'Unknown action: "{args.action}"', file=sys.stderr)
        exitcode = 2
    sys.exit(exitcode)
Exemplo n.º 7
0
 def test_icontract_snapshots(self):
     messages = analyze_function(
         icontract_appender,
         DEFAULT_OPTIONS.overlay(
             analysis_kind=[AnalysisKind.icontract]),
     )
     self.assertEqual(*check_messages(
         messages, state=MessageType.POST_FAIL, line=95, column=0))
Exemplo n.º 8
0
 def test_diff_behavior_different(self) -> None:
     diffs = diff_behavior(foo1, foo3,
                           DEFAULT_OPTIONS.overlay(max_iterations=10))
     self.assertEqual(len(diffs), 1)
     diff = diffs[0]
     assert isinstance(diff, BehaviorDiff)
     self.assertGreater(int(diff.args["x"]), 1000)
     self.assertEqual(diff.result1.return_repr, "100")
     self.assertEqual(diff.result2.return_repr, "1000")
Exemplo n.º 9
0
 def test_icontract_snapshots(self):
     messages = analyze_function(
         icontract_appender,
         DEFAULT_OPTIONS.overlay(analysis_kind=[AnalysisKind.icontract]),
     )
     line = icontract_appender.__wrapped__.__code__.co_firstlineno + 1
     self.assertEqual(
         *check_messages(
             messages, state=MessageType.POST_FAIL, line=line, column=0
         )
     )
Exemplo n.º 10
0
 def test_asserts(self):
     messages = analyze_function(
         remove_smallest_with_asserts,
         DEFAULT_OPTIONS.overlay(
             analysis_kind=[AnalysisKind.asserts],
             max_iterations=10,
             per_condition_timeout=5,
         ),
     )
     self.assertEqual(*check_messages(
         messages, state=MessageType.EXEC_ERR, line=85, column=0))
Exemplo n.º 11
0
 def test_diff_method(self):
     diffs = diff_behavior(
         walk_qualname(Base, "foo"),
         walk_qualname(Derived, "foo"),
         DEFAULT_OPTIONS.overlay(max_iterations=10),
     )
     assert isinstance(diffs, list)
     self.assertEqual(
         [(d.result1.return_repr, d.result2.return_repr) for d in diffs],
         [("10", "11")],
     )
Exemplo n.º 12
0
def unwalled_main(cmd_args: Union[List[str], argparse.Namespace]) -> int:
    parser = command_line_parser()
    if isinstance(cmd_args, argparse.Namespace):
        args = cmd_args
    else:
        args = parser.parse_args(cmd_args)
    if not args.action:
        parser.print_help(sys.stderr)
        return 2
    set_debug(args.verbose)
    debug("Installed plugins:", installed_plugins)
    options = option_set_from_dict(args.__dict__)
    # fall back to current directory to look up modules
    with add_to_pypath(*([""] if sys.path and sys.path[0] != "" else [])):
        if args.action == "check":
            return check(args, options, sys.stdout, sys.stderr)
        elif args.action == "diffbehavior":
            defaults = DEFAULT_OPTIONS.overlay(
                AnalysisOptionSet(
                    per_condition_timeout=2.5,
                    per_path_timeout=
                    30.0,  # mostly, we don't want to time out paths
                ))
            return diffbehavior(args, defaults.overlay(options), sys.stdout,
                                sys.stderr)
        elif args.action == "cover":
            defaults = DEFAULT_OPTIONS.overlay(
                AnalysisOptionSet(
                    per_condition_timeout=2.5,
                    per_path_timeout=
                    30.0,  # mostly, we don't want to time out paths
                ))
            return cover(args, defaults.overlay(options), sys.stdout,
                         sys.stderr)
        elif args.action == "watch":
            return watch(args, options)
        else:
            print(f'Unknown action: "{args.action}"', file=sys.stderr)
            return 2
Exemplo n.º 13
0
def run_watch_loop(watcher, max_watch_iterations=sys.maxsize) -> None:
    restart = True
    stats: Counter[str] = Counter()
    active_messages: Dict[Tuple[str, int], AnalysisMessage]
    for itr_num in range(max_watch_iterations):
        if restart:
            clear_screen()
            clear_line("-")
            line = f"  Analyzing {len(watcher._modtimes)} files."
            sys.stdout.write(color(line, AnsiColor.OKBLUE))
            sys.stdout.flush()
            max_condition_timeout = 0.5
            restart = False
            stats = Counter()
            active_messages = {}
        else:
            time.sleep(0.5)
            max_condition_timeout *= 2
        for curstats, messages in watcher.run_iteration(max_condition_timeout):
            debug("stats", curstats, messages)
            stats.update(curstats)
            clear_screen()
            if messages_merged(active_messages, messages):
                linecache.checkcache()
            options = DEFAULT_OPTIONS.overlay(watcher._options)
            for message in active_messages.values():
                lines = long_describe_message(message, options)
                if lines is None:
                    continue
                clear_line("-")
                print(lines, end="")
            clear_line("-")
            num_files = len(watcher._modtimes)
            if len(watcher._paths) > 1:
                loc_desc = f"{num_files} files"
            else:
                path_desc = Path(next(iter(watcher._paths))).parts[-1]
                if num_files > 1:
                    loc_desc = f'"{path_desc}" ({num_files} files)'
                else:
                    loc_desc = f'"{path_desc}"'
            line = f'  Analyzed {stats["num_paths"]} paths in {loc_desc}.'
            sys.stdout.write(color(line, AnsiColor.OKBLUE))
            sys.stdout.flush()
        if watcher._change_flag:
            watcher._change_flag = False
            restart = True
            line = f"  Restarting analysis over {len(watcher._modtimes)} files."
            sys.stdout.write(color(line, AnsiColor.OKBLUE))
            sys.stdout.flush()
Exemplo n.º 14
0
 def test_hypothesis_counterexample_text():
     messages = analyze_function(
         foo,
         DEFAULT_OPTIONS.overlay(
             analysis_kind=[AnalysisKind.hypothesis],
             max_iterations=10,
             per_condition_timeout=20,
             per_path_timeout=5,
         ),
     )
     actual, expected = check_messages(
         messages,
         state=MessageType.EXEC_ERR,
         message="AssertionError: assert False when calling foo(x = False)",
     )
     assert actual == expected
Exemplo n.º 15
0
    def test_diff_behavior_mutation(self) -> None:
        def cut_out_item1(a: List[int], i: int):
            a[i:i + 1] = []

        def cut_out_item2(a: List[int], i: int):
            a[:] = a[:i] + a[i + 1:]

        # TODO: this takes longer than I'd like (few iterations though):
        opts = DEFAULT_OPTIONS.overlay(max_iterations=20,
                                       per_path_timeout=10,
                                       per_condition_timeout=10)
        diffs = diff_behavior(
            FunctionInfo.from_fn(cut_out_item1),
            FunctionInfo.from_fn(cut_out_item2),
            opts,
        )
        assert not isinstance(diffs, str)
        self.assertEqual(len(diffs), 1)
        diff = diffs[0]
        self.assertGreater(len(diff.args["a"]), 1)
        self.assertEqual(diff.args["i"], "-1")
Exemplo n.º 16
0
 def run_watch_loop(self, max_watch_iterations=sys.maxsize) -> None:
     restart = True
     stats: Counter[str] = Counter()
     active_messages: Dict[Tuple[str, int], AnalysisMessage]
     for itr_num in range(max_watch_iterations):
         if restart:
             clear_screen()
             clear_line("-")
             line = f"  Analyzing {len(self._modtimes)} files.          \r"
             sys.stdout.write(color(line, AnsiColor.OKBLUE))
             max_condition_timeout = 0.5
             restart = False
             stats = Counter()
             active_messages = {}
         else:
             time.sleep(0.5)
             max_condition_timeout *= 2
         for curstats, messages in self.run_iteration(
                 max_condition_timeout):
             debug("stats", curstats, messages)
             stats.update(curstats)
             if messages_merged(active_messages, messages):
                 linecache.checkcache()
                 clear_screen()
                 options = DEFAULT_OPTIONS.overlay(self._options)
                 for message in active_messages.values():
                     lines = long_describe_message(message, options)
                     if lines is None:
                         continue
                     clear_line("-")
                     print(lines, end="")
                 clear_line("-")
             line = f'  Analyzed {stats["num_paths"]} paths in {len(self._modtimes)} files.          \r'
             sys.stdout.write(color(line, AnsiColor.OKBLUE))
         if self._change_flag:
             self._change_flag = False
             restart = True
             line = f"  Restarting analysis over {len(self._modtimes)} files.          \r"
             sys.stdout.write(color(line, AnsiColor.OKBLUE))
Exemplo n.º 17
0
from crosshair.path_cover import CoverageType
from crosshair.core_and_libs import *


def _foo(x: int) -> int:
    if x > 100:
        return 100
    return x


def _regex(x: str) -> bool:
    compiled = re.compile("f(o)+")
    return bool(compiled.fullmatch(x))


OPTS = DEFAULT_OPTIONS.overlay(max_iterations=10, per_condition_timeout=10.0)
foo = FunctionInfo.from_fn(_foo)
regex = FunctionInfo.from_fn(_regex)


def test_path_cover() -> None:
    paths = list(path_cover(foo, OPTS, CoverageType.OPCODE))
    assert len(paths) == 2
    small, large = sorted(paths, key=lambda p: p.result)  # type: ignore
    assert large.result == 100
    assert large.args.arguments["x"] > 100
    assert small.result == small.args.arguments["x"]


def test_path_cover_regex() -> None:
    paths = list(path_cover(regex, OPTS, CoverageType.OPCODE))
Exemplo n.º 18
0
 def test_diff_behavior_same(self) -> None:
     diffs = diff_behavior(foo1, foo2,
                           DEFAULT_OPTIONS.overlay(max_iterations=10))
     self.assertEqual(diffs, [])
Exemplo n.º 19
0
def run_watch_loop(
    watcher: Watcher,
    max_watch_iterations: int = sys.maxsize,
    term_lines_rewritable: bool = True,
) -> None:
    restart = True
    stats: Counter[str] = Counter()
    active_messages: Dict[Tuple[str, int], AnalysisMessage]
    for _ in range(max_watch_iterations):
        if restart:
            clear_screen()
            print_divider("-")
            line = f"  Analyzing {len(watcher._modtimes)} files."
            print(color(line, AnsiColor.OKBLUE), end="")
            max_condition_timeout = 0.5
            restart = False
            stats = Counter()
            active_messages = {}
        else:
            time.sleep(0.1)
            max_condition_timeout *= 2
        for curstats, messages in watcher.run_iteration(max_condition_timeout):
            messages = [m for m in messages if m.state > MessageType.PRE_UNSAT]
            stats.update(curstats)
            if messages_merged(active_messages, messages):
                linecache.checkcache()
                clear_screen()
                options = DEFAULT_OPTIONS.overlay(watcher._options)
                for message in active_messages.values():
                    lines = long_describe_message(message, options)
                    if lines is None:
                        continue
                    print_divider("-")
                    print(lines, end="")
                print_divider("-")
            else:
                if term_lines_rewritable:
                    print("\r", end="")
                else:
                    print(".", end="")
                    continue
            num_files = len(watcher._modtimes)
            if len(watcher._paths) > 1:
                loc_desc = f"{num_files} files"
            else:
                path_parts = Path(next(iter(watcher._paths))).parts
                path_desc = path_parts[-1] if path_parts else "."
                if num_files > 1:
                    loc_desc = f'"{path_desc}" ({num_files} files)'
                else:
                    loc_desc = f'"{path_desc}"'
            if term_lines_rewritable:
                line = f'  Analyzed {stats["num_paths"]} paths in {loc_desc}.       '
            else:
                line = f"  Analyzing paths in {loc_desc}: "
            print(color(line, AnsiColor.OKBLUE), end="")
            if watcher._change_flag:
                watcher._change_flag = False
                restart = True
                line = f"  Restarting analysis over {len(watcher._modtimes)} files."
                print(color(line, AnsiColor.OKBLUE), end="")