def test_example_coverage(self) -> None:
# Try to get examples that highlist the differences in the code.
# Here, we add more conditions for the `return True` path and
# another case where we used to just `return False`.
def isack1(s: str) -> bool:
if s in ("y", "yes"):
return True
return False
def isack2(s: str) -> Optional[bool]:
if s in ("y", "yes", "Y", "YES"):
return True
if s in ("n", "no", "N", "NO"):
return False
return None
diffs = diff_behavior(
FunctionInfo.from_fn(isack1),
FunctionInfo.from_fn(isack2),
DEFAULT_OPTIONS.overlay(max_iterations=20,
per_condition_timeout=5),
)
debug("diffs=", diffs)
assert not isinstance(diffs, str)
return_vals = set(
(d.result1.return_repr, d.result2.return_repr) for d in diffs)
self.assertEqual(return_vals, {("False", "None"), ("False", "True")})
def diff_behavior(ctxfn1: FunctionInfo, ctxfn2: FunctionInfo,
options: AnalysisOptions) -> Union[str, List[BehaviorDiff]]:
fn1, sig1 = ctxfn1.callable()
fn2, sig2 = ctxfn2.callable()
debug("Resolved signature:", sig1)
all_diffs: List[BehaviorDiff] = []
half1, half2 = options.split_limits(0.5)
with condition_parser(options.analysis_kind), Patched():
# We attempt both orderings of functions. This helps by:
# (1) avoiding code path explosions in one of the functions
# (2) using both signatures (in case they differ)
all_diffs.extend(diff_behavior_with_signature(fn1, fn2, sig1, half1))
all_diffs.extend(
diff.reverse()
for diff in diff_behavior_with_signature(fn2, fn1, sig2, half2))
debug("diff candidates:", all_diffs)
# greedily pick results:
result_diffs = []
opcodeset1 = set(i.offset for i in dis.get_instructions(fn1.__code__))
opcodeset2 = set(i.offset for i in dis.get_instructions(fn2.__code__))
while all_diffs:
scorer = diff_scorer(opcodeset1, opcodeset2)
selection = max(all_diffs, key=scorer)
(num_opcodes, _) = scorer(selection)
debug("Considering input", selection.args, " num opcodes=",
num_opcodes)
if num_opcodes == 0:
break
all_diffs.remove(selection)
result_diffs.append(selection)
coverage1, coverage2 = selection.coverage1, selection.coverage2
if coverage1 is not None and coverage2 is not None:
opcodeset1 -= coverage1.offsets_covered
opcodeset2 -= coverage2.offsets_covered
return result_diffs
def test_CompositeConditionParser():
composite = CompositeConditionParser()
composite.parsers.append(Pep316Parser(composite))
composite.parsers.append(AssertsParser(composite))
assert composite.get_fn_conditions(
FunctionInfo.from_fn(single_line_condition)
).has_any()
assert composite.get_fn_conditions(FunctionInfo.from_fn(avg_with_asserts)).has_any()
def test_implies_condition(self):
conditions = Pep316Parser().get_fn_conditions(
FunctionInfo.from_fn(implies_condition)
)
assert conditions is not None
# This shouldn't explode (avoid a KeyError on record['override']):
conditions.post[0].evaluate({"record": {}, "_": 0})
def test_tricky_raises_condition(self) -> None:
conditions = Pep316Parser().get_fn_conditions(
FunctionInfo.from_fn(tricky_raises_condition)
)
assert conditions is not None
self.assertEqual([], list(conditions.syntax_messages()))
self.assertEqual(set([KeyError, OSError]), conditions.raises)
def test_locally_defined_raises_condition(self) -> None:
conditions = Pep316Parser().get_fn_conditions(
FunctionInfo.from_fn(locally_defined_raises_condition)
)
assert conditions is not None
self.assertEqual([], list(conditions.syntax_messages()))
self.assertEqual(set([LocallyDefiendException]), conditions.raises)
def trace_call(
self,
frame: FrameType,
fn: Callable,
binding_target: object,
) -> Optional[Callable]:
caller_code = frame.f_code
if not self.cached_wants_codeobj(caller_code):
return None
target_name = getattr(fn, "__name__", "")
if target_name.endswith((">", "_crosshair_wrapper")):
return None
if isinstance(fn, NoEnforce):
return fn.fn
if type(fn) is type and fn not in (super, type):
return functools.partial(manually_construct, fn)
condition_parser = self.condition_parser
# TODO: Is doing this a problem? A method's function's conditions depend on the
# class of self.
ctxfn = FunctionInfo(None, "", fn) # type: ignore
conditions = condition_parser.get_fn_conditions(ctxfn)
if conditions is not None and not conditions.has_any():
conditions = None
if conditions is None:
return None
# debug("Enforcing conditions on", fn, 'binding', binding_target)
fn = self.interceptor(conditions.fn) # type: ignore
is_bound = binding_target is not None
wrapper = EnforcementWrapper(fn, conditions, self,
binding_target) # type: ignore
return wrapper
def trace_call(
self,
frame: FrameType,
fn: Callable,
binding_target: object,
) -> Optional[Callable]:
caller_code = frame.f_code
if caller_code.co_name == "_crosshair_wrapper":
return None
target_name = getattr(fn, "__name__", "")
if target_name.endswith((">", "_crosshair_wrapper")):
return None
if isinstance(fn, NoEnforce):
return fn.fn
condition_parser = self.condition_parser
# TODO: Remove the FunctionInfo concept
ctxfn = FunctionInfo(None, "", fn) # type: ignore
conditions = condition_parser.get_fn_conditions(ctxfn)
if conditions is not None and not conditions.has_any():
conditions = None
if conditions is None:
return None
# debug("Enforcing conditions on", fn, 'binding', binding_target)
fn = self.interceptor(conditions.fn) # type: ignore
is_bound = binding_target is not None
wrapper = EnforcementWrapper(fn, conditions, self,
binding_target) # type: ignore
return wrapper
def get_class_conditions(self, cls: type) -> ClassConditions:
if not is_pure_python(cls):
# We can't get conditions/line numbers for classes written in C.
return ClassConditions([], {})
toplevel_parser = self.get_toplevel_parser()
methods = {}
super_conditions = merge_class_conditions([
toplevel_parser.get_class_conditions(base)
for base in cls.__bases__
])
inv = self.get_class_invariants(cls)
super_methods = super_conditions.methods
method_names = set(cls.__dict__.keys()) | super_methods.keys()
for method_name in method_names:
method = cls.__dict__.get(method_name, None)
super_method_conditions = super_methods.get(method_name)
if super_method_conditions is not None:
revised_sig = set_first_arg_type(super_method_conditions.sig,
cls)
super_method_conditions = replace(super_method_conditions,
sig=revised_sig)
if method is None:
if super_method_conditions is None:
continue
else:
conditions: Conditions = super_method_conditions
else:
parsed_conditions = toplevel_parser.get_fn_conditions(
FunctionInfo.from_class(cls, method_name))
if parsed_conditions is None:
# debug(f'Skipping "{method_name}": Unable to determine the function signature.')
continue
if super_method_conditions is None:
conditions = parsed_conditions
else:
conditions = merge_fn_conditions(parsed_conditions,
super_method_conditions)
if method_name in ("__new__", "__repr__"):
# __new__ isn't passed a concrete instance.
# __repr__ is itself required for reporting problems with invariants.
use_pre, use_post = False, False
elif method_name == "__del__":
use_pre, use_post = True, False
elif method_name == "__init__":
use_pre, use_post = False, True
elif method_name.startswith("__") and method_name.endswith("__"):
use_pre, use_post = True, True
elif method_name.startswith("_"):
use_pre, use_post = False, False
else:
use_pre, use_post = True, True
if use_pre:
conditions.pre.extend(inv)
if use_post:
conditions.post.extend(inv)
if conditions.has_any():
methods[method_name] = conditions
return ClassConditions(inv, methods)
def test_CompositeConditionParser_adds_completion_conditions():
composite_parser = CompositeConditionParser()
pep316_parser = Pep316Parser(composite_parser)
composite_parser.parsers.append(pep316_parser)
fn = FunctionInfo.from_fn(no_postconditions)
assert len(pep316_parser.get_fn_conditions(fn).pre) == 1
assert len(pep316_parser.get_fn_conditions(fn).post) == 0
assert len(composite_parser.get_fn_conditions(fn).post) == 1
def test_single_line_condition(self) -> None:
conditions = Pep316Parser().get_fn_conditions(
FunctionInfo.from_fn(single_line_condition)
)
assert conditions is not None
self.assertEqual(
set([c.expr_source for c in conditions.post]), set(["__return__ >= x"])
)
def tests_simple_parse(self) -> None:
conditions = AssertsParser().get_fn_conditions(
FunctionInfo.from_fn(avg_with_asserts))
assert conditions is not None
conditions.fn([])
self.assertEqual(conditions.fn([2.2]), 2.2)
with self.assertRaises(AssertionError):
conditions.fn([9.2, 17.8])
def get_fn_conditions(self, ctxfn: FunctionInfo) -> Optional[Conditions]:
fn_and_sig = ctxfn.get_callable()
if fn_and_sig is None:
return None
(fn, sig) = fn_and_sig
filename, first_line, _lines = sourcelines(fn)
if isinstance(fn, types.BuiltinFunctionType):
return Conditions(fn, fn, [], [], frozenset(), sig, frozenset(),
[])
lines = list(get_doc_lines(fn))
parse = parse_sections(lines, ("pre", "post", "raises"), filename)
pre: List[ConditionExpr] = []
raises: Set[Type[BaseException]] = set()
post_conditions: List[ConditionExpr] = []
mutable_args: Optional[FrozenSet[str]] = None
if parse.mutable_expr is not None:
mutable_args = frozenset(expr.strip().split(".")[0]
for expr in parse.mutable_expr.split(",")
if expr != "")
for line_num, expr in parse.sections["pre"]:
pre.append(
condition_from_source_text(filename, line_num, expr,
fn_globals(fn)))
for line_num, expr in parse.sections["raises"]:
if "#" in expr:
expr = expr.split("#")[0]
for exc_source in expr.split(","):
try:
exc_type = eval(exc_source)
except:
e = sys.exc_info()[1]
parse.syntax_messages.append(
ConditionSyntaxMessage(filename, line_num, str(e)))
continue
if not issubclass(exc_type, BaseException):
parse.syntax_messages.append(
ConditionSyntaxMessage(
filename,
line_num,
f'"{exc_type}" is not an exception class',
))
continue
raises.add(exc_type)
for line_num, expr in parse.sections["post"]:
post_conditions.append(
condition_from_source_text(filename, line_num, expr,
fn_globals(fn)))
return Conditions(
fn,
fn,
pre,
post_conditions,
frozenset(raises),
sig,
mutable_args,
parse.syntax_messages,
)
def test_conditions_with_closure_references_and_string_type(self) -> None:
# This is a function that refers to something in its closure.
# Ensure we can still look up string-based types:
def referenced_fn():
return 4
def fn_with_closure(foo: "Foo"):
referenced_fn()
# Ensure we don't error trying to resolve "Foo":
Pep316Parser().get_fn_conditions(FunctionInfo.from_fn(fn_with_closure))
def test_diff_behavior_mutation(self) -> None:
def cut_out_item1(a: List[int], i: int):
a[i:i + 1] = []
def cut_out_item2(a: List[int], i: int):
a[:] = a[:i] + a[i + 1:]
# TODO: this takes longer than I'd like (few iterations though):
opts = DEFAULT_OPTIONS.overlay(max_iterations=20,
per_path_timeout=10,
per_condition_timeout=10)
diffs = diff_behavior(
FunctionInfo.from_fn(cut_out_item1),
FunctionInfo.from_fn(cut_out_item2),
opts,
)
assert not isinstance(diffs, str)
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertGreater(len(diff.args["a"]), 1)
self.assertEqual(diff.args["i"], "-1")
def path_cover(ctxfn: FunctionInfo, options: AnalysisOptions,
coverage_type: CoverageType) -> List[PathSummary]:
fn, sig = ctxfn.callable()
search_root = SinglePathNode(True)
condition_start = time.monotonic()
paths: List[PathSummary] = []
for i in range(1, options.max_iterations):
debug("Iteration ", i)
itr_start = time.monotonic()
if itr_start > condition_start + options.per_condition_timeout:
debug(
"Stopping due to --per_condition_timeout=",
options.per_condition_timeout,
)
break
space = StateSpace(
execution_deadline=itr_start + options.per_path_timeout,
model_check_timeout=options.per_path_timeout / 2,
search_root=search_root,
)
with condition_parser(options.analysis_kind), Patched(
), COMPOSITE_TRACER, StateSpaceContext(space):
summary = None
try:
summary = run_iteration(fn, sig, space)
verification_status = VerificationStatus.CONFIRMED
except UnexploredPath:
verification_status = VerificationStatus.UNKNOWN
debug("Verification status:", verification_status)
top_analysis, exhausted = space.bubble_status(
CallAnalysis(verification_status))
debug("Path tree stats", search_root.stats())
if summary:
paths.append(summary)
if exhausted:
debug("Stopping due to code path exhaustion. (yay!)")
break
opcodes_found: Set[int] = set()
selected: List[PathSummary] = []
while paths:
next_best = max(
paths,
key=lambda p: len(p.coverage.offsets_covered - opcodes_found))
cur_offsets = next_best.coverage.offsets_covered
if coverage_type == CoverageType.OPCODE:
if len(cur_offsets - opcodes_found) == 0:
break
selected.append(next_best)
opcodes_found |= cur_offsets
paths = [p for p in paths if p is not next_best]
return selected
def get_fn_conditions(self, ctxfn: FunctionInfo) -> Optional[Conditions]:
fn_and_sig = ctxfn.get_callable()
if fn_and_sig is None:
return None
(fn, sig) = fn_and_sig
# TODO replace this guard with package-level configuration?
if (getattr(fn, "__module__", False)
and fn.__module__.startswith("crosshair.")
and not fn.__module__.endswith("_test")):
return None
try:
first_body_line = AssertsParser.get_first_body_line(fn)
except OSError:
return None
if first_body_line is None:
return None
filename, first_line, _lines = sourcelines(fn)
@wraps(fn)
def wrappedfn(*a, **kw):
try:
return NoEnforce(fn)(*a, **kw)
except AssertionError as e:
# TODO: check that this isn't failing at an early line in a different
# file?
_, lineno = frame_summary_for_fn(
fn, traceback.extract_tb(e.__traceback__))
if lineno >= first_body_line:
raise
post = [
ConditionExpr(
POSTCONDIITON,
lambda _: True,
filename,
first_line,
"",
)
]
return Conditions(
wrappedfn,
fn,
[], # (pre)
post,
raises=frozenset(parse_sphinx_raises(fn)),
sig=sig,
mutable_args=None,
fn_syntax_messages=[],
)
def tests_extra_ast_nodes(self) -> None:
conditions = AssertsParser().get_fn_conditions(
FunctionInfo.from_fn(fn_with_docstring_comments_and_assert))
assert conditions is not None
# Empty list does not pass precondition, ignored:
conditions.fn([])
# normal, passing case:
nums = [3, 1, 2]
conditions.fn(nums)
self.assertEqual(nums, [3, 2])
# Failing case (duplicate minimum values):
with self.assertRaises(AssertionError):
nums = [3, 1, 1, 2]
conditions.fn(nums)
def _wrap_fn(self,
fn: Callable,
conditions: Optional[Conditions] = None) -> Callable:
wrapper = self.wrapper_map.get(fn)
if wrapper is not None:
return wrapper
if conditions is None:
conditions = self.condition_parser.get_fn_conditions(
FunctionInfo.from_fn(fn)) # type: ignore
if conditions and conditions.has_any():
wrapper = EnforcementWrapper(self.interceptor(fn), conditions,
self)
functools.update_wrapper(wrapper, fn)
else:
wrapper = fn
self.wrapper_map[fn] = wrapper
self.original_map[IdentityWrapper(wrapper)] = fn
return wrapper
def test_simple_parse(self):
@icontract.require(lambda l: len(l) > 0)
@icontract.ensure(lambda l, result: min(l) <= result <= max(l))
def avg(l):
return sum(l) / len(l)
conditions = IcontractParser().get_fn_conditions(FunctionInfo.from_fn(avg))
assert conditions is not None
self.assertEqual(len(conditions.pre), 1)
self.assertEqual(len(conditions.post), 1)
self.assertEqual(conditions.pre[0].evaluate({"l": []}), False)
post_args = {
"l": [42, 43],
"__old__": AttributeHolder({}),
"__return__": 40,
"_": 40,
}
self.assertEqual(conditions.post[0].evaluate(post_args), False)
self.assertEqual(len(post_args), 4) # (check args are unmodified)
def get_fn_conditions(self, ctxfn: FunctionInfo) -> Optional[Conditions]:
fn_and_sig = ctxfn.get_callable()
if fn_and_sig is None:
return None
(fn, sig) = fn_and_sig
if not getattr(fn, "is_hypothesis_test", False):
return None
fuzz_one = getattr(getattr(fn, "hypothesis", None), "fuzz_one_input",
None)
if fuzz_one is None:
return None
filename, first_line, _lines = sourcelines(fn)
post = [
ConditionExpr(
POSTCONDIITON,
lambda _: True,
filename,
first_line,
"",
)
]
sig = inspect.Signature(parameters=[
inspect.Parameter(
"payload", inspect.Parameter.POSITIONAL_ONLY, annotation=bytes)
])
return Conditions(
fuzz_one,
fn,
[], # (pre)
post,
raises=frozenset(),
sig=sig,
mutable_args=None,
fn_syntax_messages=[],
counterexample_description_maker=partial(
self._format_counterexample, fn),
)
def _wrap_class_members(self, cls: type,
class_conditions: ClassConditions) -> None:
method_conditions = dict(class_conditions.methods)
for method_name in list(cls.__dict__.keys()):
conditions = method_conditions.get(method_name)
if conditions is None:
continue
ctxfn = FunctionInfo.from_class(cls, method_name)
raw_fn = ctxfn.descriptor
wrapper = self.wrapper_map.get(raw_fn)
if wrapper is None:
if conditions.has_any():
fn, _ = ctxfn.callable()
wrapper = EnforcementWrapper(self.interceptor(fn),
conditions, self)
functools.update_wrapper(wrapper, fn)
else:
wrapper = fn
self.wrapper_map[raw_fn] = wrapper
outer_wrapper = ctxfn.patch_logic(wrapper)
self.original_map[IdentityWrapper(outer_wrapper)] = raw_fn
setattr(cls, method_name, outer_wrapper)
def test_builtin_conditions_are_null(self) -> None:
self.assertIsNone(Pep316Parser().get_fn_conditions(FunctionInfo.from_fn(zip)))
def tests_empty_parse(self) -> None:
conditions = AssertsParser().get_fn_conditions(FunctionInfo.from_fn(debug))
self.assertEqual(conditions, None)
from crosshair.diff_behavior import diff_behavior
from crosshair.fnutil import walk_qualname
from crosshair.fnutil import FunctionInfo
from crosshair.options import AnalysisOptions
from crosshair.options import DEFAULT_OPTIONS
from crosshair.util import debug
from crosshair.util import set_debug
def _foo1(x: int) -> int:
if x >= 100:
return 100
return x
foo1 = FunctionInfo.from_fn(_foo1)
def _foo2(x: int) -> int:
return min(x, 100)
foo2 = FunctionInfo.from_fn(_foo2)
def _foo3(x: int) -> int:
if x > 1000:
return 1000
elif x > 100:
return 100
else:
def get_fn_conditions(self, ctxfn: FunctionInfo) -> Optional[Conditions]:
if icontract is None:
return None
fn_and_sig = ctxfn.get_callable()
if fn_and_sig is None:
return None
(fn, sig) = fn_and_sig
checker = icontract._checkers.find_checker(func=fn) # type: ignore
contractless_fn = fn # type: ignore
while (hasattr(contractless_fn, "__is_invariant_check__")
or hasattr(contractless_fn, "__preconditions__")
or hasattr(contractless_fn, "__postconditions__")):
contractless_fn = contractless_fn.__wrapped__ # type: ignore
if checker is None:
return Conditions(contractless_fn, contractless_fn, [], [],
frozenset(), sig, None, [])
pre: List[ConditionExpr] = []
post: List[ConditionExpr] = []
def eval_contract(contract, kwargs):
condition_kwargs = icontract._checkers.select_condition_kwargs(
contract=contract, resolved_kwargs=kwargs)
return contract.condition(**condition_kwargs)
disjunction = checker.__preconditions__ # type: ignore
if len(disjunction) == 0:
pass
elif len(disjunction) == 1:
for contract in disjunction[0]:
evalfn = functools.partial(eval_contract, contract)
filename, line_num, _lines = sourcelines(contract.condition)
pre.append(
ConditionExpr(evalfn, filename, line_num,
self.contract_text(contract)))
else:
def eval_disjunction(disjunction, kwargs) -> bool:
for conjunction in disjunction:
ok = True
for contract in conjunction:
if not eval_contract(contract, kwargs):
ok = False
break
if ok:
return True
return False
evalfn = functools.partial(eval_disjunction, disjunction)
filename, line_num, _lines = sourcelines(contractless_fn)
source = ("(" + ") or (".join([
" and ".join([self.contract_text(c) for c in conj])
for conj in disjunction
]) + ")")
pre.append(ConditionExpr(evalfn, filename, line_num, source))
snapshots = checker.__postcondition_snapshots__ # type: ignore
def take_snapshots(**kwargs):
old_as_mapping: MutableMapping[str, Any] = {}
for snap in snapshots:
snap_kwargs = icontract._checkers.select_capture_kwargs(
a_snapshot=snap, resolved_kwargs=kwargs)
old_as_mapping[snap.name] = snap.capture(**snap_kwargs)
return icontract._checkers.Old(mapping=old_as_mapping)
def post_eval(contract, kwargs):
_old = kwargs.pop("__old__")
kwargs["OLD"] = take_snapshots(**_old.__dict__)
kwargs["result"] = kwargs.pop("__return__")
del kwargs["_"]
condition_kwargs = icontract._checkers.select_condition_kwargs(
contract=contract, resolved_kwargs=kwargs)
return contract.condition(**condition_kwargs)
for postcondition in checker.__postconditions__: # type: ignore
evalfn = functools.partial(post_eval, postcondition)
filename, line_num, _lines = sourcelines(postcondition.condition)
post.append(
ConditionExpr(evalfn, filename, line_num,
self.contract_text(postcondition)))
return Conditions(
contractless_fn,
contractless_fn,
pre,
post,
raises=frozenset(parse_sphinx_raises(fn)),
sig=sig,
mutable_args=None,
fn_syntax_messages=[],
)
from crosshair.core_and_libs import *
def _foo(x: int) -> int:
if x > 100:
return 100
return x
def _regex(x: str) -> bool:
compiled = re.compile("f(o)+")
return bool(compiled.fullmatch(x))
OPTS = DEFAULT_OPTIONS.overlay(max_iterations=10, per_condition_timeout=10.0)
foo = FunctionInfo.from_fn(_foo)
regex = FunctionInfo.from_fn(_regex)
def test_path_cover() -> None:
paths = list(path_cover(foo, OPTS, CoverageType.OPCODE))
assert len(paths) == 2
small, large = sorted(paths, key=lambda p: p.result) # type: ignore
assert large.result == 100
assert large.args.arguments["x"] > 100
assert small.result == small.args.arguments["x"]
def test_path_cover_regex() -> None:
paths = list(path_cover(regex, OPTS, CoverageType.OPCODE))
assert len(paths) == 1