def run_iteration(
fn1: Callable, fn2: Callable, sig: inspect.Signature, space: StateSpace
) -> Tuple[Optional[VerificationStatus], Optional[BehaviorDiff]]:
original_args = gen_args(sig)
args1 = copy.deepcopy(original_args)
args2 = copy.deepcopy(original_args)
coverage_manager = measure_fn_coverage(fn1, fn2)
with ExceptionFilter() as efilter, coverage_manager as coverage:
result1 = describe_behavior(fn1, args1)
result2 = describe_behavior(fn2, args2)
space.check_deferred_assumptions()
if result1 == result2 and args1 == args2:
debug("Functions equivalent")
return (VerificationStatus.CONFIRMED, None)
debug("Functions differ")
realized_args = {
k: repr(v)
for (k, v) in original_args.arguments.items()
}
post_execution_args1 = {k: repr(v) for k, v in args1.arguments.items()}
post_execution_args2 = {k: repr(v) for k, v in args2.arguments.items()}
diff = BehaviorDiff(
realized_args,
Result(repr(result1[0]), result1[1], post_execution_args1),
Result(repr(result2[0]), result2[1], post_execution_args2),
coverage(fn1),
coverage(fn2),
)
return (VerificationStatus.REFUTED, diff)
if efilter.user_exc:
debug("User-level exception found", repr(efilter.user_exc[0]),
efilter.user_exc[1])
return (None, None)
def run_iteration(fn: Callable, sig: Signature,
space: StateSpace) -> Optional[PathSummary]:
with NoTracing():
args = gen_args(sig)
pre_args = copy.deepcopy(args)
ret = None
with measure_fn_coverage(fn) as coverage, ExceptionFilter() as efilter:
# coverage = lambda _: CoverageResult(set(), set(), 1.0)
# with ExceptionFilter() as efilter:
ret = fn(*args.args, **args.kwargs)
space.detach_path()
if efilter.user_exc is not None:
exc = efilter.user_exc[0]
debug("user-level exception found", repr(exc), *efilter.user_exc[1])
return PathSummary(pre_args, ret, type(exc), args, coverage(fn))
elif efilter.ignore:
return None
else:
return PathSummary(
deep_realize(pre_args),
deep_realize(ret),
None,
deep_realize(args),
coverage(fn),
)
def choose_type(space: StateSpace, from_type: Type) -> Type:
subtypes = get_subclass_map()[from_type]
# Note that this is written strangely to leverage the default
# preference for false when forking:
if not subtypes or not space.smt_fork():
return from_type
for subtype in subtypes[:-1]:
if not space.smt_fork():
return choose_type(space, subtype)
return choose_type(space, subtypes[-1])
def path_cover(ctxfn: FunctionInfo, options: AnalysisOptions,
coverage_type: CoverageType) -> List[PathSummary]:
fn, sig = ctxfn.callable()
search_root = SinglePathNode(True)
condition_start = time.monotonic()
paths: List[PathSummary] = []
for i in range(1, options.max_iterations):
debug("Iteration ", i)
itr_start = time.monotonic()
if itr_start > condition_start + options.per_condition_timeout:
debug(
"Stopping due to --per_condition_timeout=",
options.per_condition_timeout,
)
break
space = StateSpace(
execution_deadline=itr_start + options.per_path_timeout,
model_check_timeout=options.per_path_timeout / 2,
search_root=search_root,
)
with condition_parser(options.analysis_kind), Patched(
), COMPOSITE_TRACER, StateSpaceContext(space):
summary = None
try:
summary = run_iteration(fn, sig, space)
verification_status = VerificationStatus.CONFIRMED
except UnexploredPath:
verification_status = VerificationStatus.UNKNOWN
debug("Verification status:", verification_status)
top_analysis, exhausted = space.bubble_status(
CallAnalysis(verification_status))
debug("Path tree stats", search_root.stats())
if summary:
paths.append(summary)
if exhausted:
debug("Stopping due to code path exhaustion. (yay!)")
break
opcodes_found: Set[int] = set()
selected: List[PathSummary] = []
while paths:
next_best = max(
paths,
key=lambda p: len(p.coverage.offsets_covered - opcodes_found))
cur_offsets = next_best.coverage.offsets_covered
if coverage_type == CoverageType.OPCODE:
if len(cur_offsets - opcodes_found) == 0:
break
selected.append(next_best)
opcodes_found |= cur_offsets
paths = [p for p in paths if p is not next_best]
return selected
def symbolic_run(
self,
fn: Callable[[StateSpace, Dict[str, object]], object],
typed_args: Dict[str, type],
) -> Tuple[object, # return value
Optional[Dict[str, object]], # arguments after execution
Optional[BaseException], # exception thrown, if any
StateSpace, ]:
search_root = SinglePathNode(True)
with COMPOSITE_TRACER, Patched():
for itr in range(1, 200):
debug("iteration", itr)
space = StateSpace(time.monotonic() + 10.0,
1.0,
search_root=search_root)
symbolic_args = {}
try:
with StateSpaceContext(space):
symbolic_args = {
name: proxy_for_type(typ, name)
for name, typ in typed_args.items()
}
ret = fn(space, symbolic_args)
ret = (deep_realize(ret), symbolic_args, None, space)
space.check_deferred_assumptions()
return ret
except IgnoreAttempt as e:
debug("ignore iteration attempt: ", str(e))
pass
except BaseException as e:
debug(traceback.format_exc())
return (None, symbolic_args, e, space)
top_analysis, space_exhausted = space.bubble_status(
CallAnalysis())
if space_exhausted:
return (
None,
symbolic_args,
CrosshairInternal(f"exhausted after {itr} iterations"),
space,
)
return (
None,
None,
CrosshairInternal(
"Unable to find a successful symbolic execution"),
space,
)
def proxy_for_type(typ: Type,
space: StateSpace,
varname: str,
meet_class_invariants=True,
allow_subtypes=False) -> object:
typ = normalize_pytype(typ)
origin = origin_of(typ)
type_args = type_args_of(typ)
# special cases
if isinstance(typ, type) and issubclass(typ, enum.Enum):
enum_values = list(typ) # type:ignore
for enum_value in enum_values[:-1]:
if space.smt_fork():
return enum_value
return enum_values[-1]
proxy_factory = _SIMPLE_PROXIES.get(origin)
if proxy_factory:
def recursive_proxy_factory(t: Type):
return proxy_for_type(t,
space,
varname + space.uniq(),
allow_subtypes=allow_subtypes)
recursive_proxy_factory.space = space # type: ignore
recursive_proxy_factory.pytype = typ # type: ignore
recursive_proxy_factory.varname = varname # type: ignore
return proxy_factory(recursive_proxy_factory, *type_args)
if allow_subtypes and typ is not object:
typ = choose_type(space, typ)
return proxy_for_class(typ, space, varname, meet_class_invariants)
def pick_code(space: StateSpace) -> Tuple[str, int, int]:
last_idx = len(INT_TYPE_BOUNDS) - 1
for (idx, (code, rng)) in enumerate(INT_TYPE_BOUNDS.items()):
if idx < last_idx:
if space.smt_fork(desc=f"not_{code}_array"):
continue
return (code, *rng)
assert False, "Not Reachable"
def diff_behavior_with_signature(
fn1: Callable, fn2: Callable, sig: inspect.Signature, options: AnalysisOptions
) -> Iterable[BehaviorDiff]:
search_root = SinglePathNode(True)
condition_start = time.monotonic()
for i in range(1, options.max_iterations):
debug("Iteration ", i)
itr_start = time.monotonic()
if itr_start > condition_start + options.per_condition_timeout:
debug(
"Stopping due to --per_condition_timeout=",
options.per_condition_timeout,
)
return
options.incr("num_paths")
space = StateSpace(
execution_deadline=itr_start + options.per_path_timeout,
model_check_timeout=options.per_path_timeout / 2,
search_root=search_root,
)
with StateSpaceContext(space):
output = None
try:
(verification_status, output) = run_iteration(fn1, fn2, sig, space)
except UnexploredPath:
verification_status = VerificationStatus.UNKNOWN
debug("Verification status:", verification_status)
top_analysis, space_exhausted = space.bubble_status(
CallAnalysis(verification_status)
)
if (
top_analysis
and top_analysis.verification_status == VerificationStatus.CONFIRMED
):
debug("Stopping due to code path exhaustion. (yay!)")
options.incr("exhaustion")
break
if output:
yield output
def make_fake_object(statespace: StateSpace, cls: type,
varname: str) -> object:
constructor = get_smt_proxy_type(cls)
debug(constructor)
try:
proxy = constructor()
except TypeError as e:
# likely the type has a __new__ that expects arguments
raise CrosshairUnsupported(f'Unable to proxy {name_of_type(cls)}: {e}')
for name, typ in get_type_hints(cls).items():
origin = getattr(typ, '__origin__', None)
if origin is Callable:
continue
value = proxy_for_type(typ, statespace,
varname + '.' + name + statespace.uniq())
object.__setattr__(proxy, name, value)
return proxy
def gen_args(sig: inspect.Signature,
statespace: StateSpace) -> inspect.BoundArguments:
args = sig.bind_partial()
for param in sig.parameters.values():
smt_name = param.name + statespace.uniq()
proxy_maker = lambda typ, **kw: proxy_for_type(
typ, statespace, smt_name, allow_subtypes=True, **kw)
has_annotation = (param.annotation != inspect.Parameter.empty)
value: object
if param.kind == inspect.Parameter.VAR_POSITIONAL:
if has_annotation:
varargs_type = List[param.annotation] # type: ignore
value = proxy_maker(varargs_type)
else:
value = proxy_maker(List[Any])
elif param.kind == inspect.Parameter.VAR_KEYWORD:
if has_annotation:
varargs_type = Dict[str, param.annotation] # type: ignore
value = cast(dict, proxy_maker(varargs_type))
# Using ** on a dict requires concrete string keys. Force
# instiantiation of keys here:
value = {k.__str__(): v for (k, v) in value.items()}
else:
value = proxy_maker(Dict[str, Any])
else:
is_self = param.name == 'self'
# Object parameters should meet thier invariants iff they are not the
# class under test ("self").
meet_class_invariants = not is_self
allow_subtypes = not is_self
if has_annotation:
value = proxy_for_type(param.annotation, statespace, smt_name,
meet_class_invariants, allow_subtypes)
else:
value = proxy_for_type(cast(type, Any), statespace, smt_name,
meet_class_invariants, allow_subtypes)
debug('created proxy for', param.name, 'as type:', type(value))
args.arguments[param.name] = value
return args
def __ch_forget_contents__(self, space: StateSpace):
cls = self.__ch_pytype__()
clean = proxy_for_type(cls, space, space.uniq())
for name, val in self.__dict__.items():
self.__dict__[name] = clean.__dict__[name]
def attempt_call(conditions: Conditions, space: StateSpace, fn: Callable,
short_circuit: ShortCircuitingContext,
enforced_conditions: EnforcedConditions) -> CallAnalysis:
bound_args = gen_args(conditions.sig, space)
code_obj = fn.__code__
fn_filename, fn_start_lineno = (code_obj.co_filename,
code_obj.co_firstlineno)
try:
(lines, _) = inspect.getsourcelines(fn)
except OSError:
lines = []
fn_end_lineno = fn_start_lineno + len(lines)
def locate_msg(detail: str, suggested_filename: str,
suggested_lineno: int) -> Tuple[str, str, int, int]:
if ((os.path.abspath(suggested_filename)
== os.path.abspath(fn_filename))
and (fn_start_lineno <= suggested_lineno <= fn_end_lineno)):
return (detail, suggested_filename, suggested_lineno, 0)
else:
try:
exprline = linecache.getlines(suggested_filename)[
suggested_lineno - 1].strip()
except IndexError:
exprline = '<unknown>'
detail = f'"{exprline}" yields {detail}'
return (detail, fn_filename, fn_start_lineno, 0)
with space.framework():
original_args = copy.deepcopy(bound_args)
space.checkpoint()
lcls: Mapping[str, object] = bound_args.arguments
# In preconditions, __old__ exists but is just bound to the same args.
# This lets people write class invariants using `__old__` to, for example,
# demonstrate immutability.
lcls = {'__old__': AttributeHolder(lcls), **lcls}
expected_exceptions = conditions.raises
for precondition in conditions.pre:
with ExceptionFilter(expected_exceptions) as efilter:
with enforced_conditions.enabled_enforcement(), short_circuit:
precondition_ok = precondition.evaluate(lcls)
if not precondition_ok:
debug('Failed to meet precondition', precondition.expr_source)
return CallAnalysis(failing_precondition=precondition)
if efilter.ignore:
debug('Ignored exception in precondition.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
(user_exc, tb) = efilter.user_exc
debug('Exception attempting to meet precondition',
precondition.expr_source, ':', user_exc, tb.format())
return CallAnalysis(
failing_precondition=precondition,
failing_precondition_reason=
f'it raised "{repr(user_exc)} at {tb.format()[-1]}"')
with ExceptionFilter(expected_exceptions) as efilter:
a, kw = bound_args.args, bound_args.kwargs
with enforced_conditions.enabled_enforcement(), short_circuit:
assert not space.running_framework_code
__return__ = fn(*a, **kw)
lcls = {
**bound_args.arguments, '__return__': __return__,
'_': __return__,
'__old__': AttributeHolder(original_args.arguments),
fn.__name__: fn
}
if efilter.ignore:
debug('Ignored exception in function.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
(e, tb) = efilter.user_exc
detail = name_of_type(type(e)) + ': ' + str(e)
frame_filename, frame_lineno = frame_summary_for_fn(tb, fn)
debug('exception while evaluating function body:', detail,
frame_filename, 'line', frame_lineno)
detail += ' ' + get_input_description(space, fn.__name__,
original_args, _MISSING)
return CallAnalysis(VerificationStatus.REFUTED, [
AnalysisMessage(MessageType.EXEC_ERR,
*locate_msg(detail, frame_filename, frame_lineno),
''.join(tb.format()))
])
for argname, argval in bound_args.arguments.items():
if (conditions.mutable_args is not None
and argname not in conditions.mutable_args):
old_val, new_val = original_args.arguments[argname], argval
if not deep_eq(old_val, new_val, set()):
detail = 'Argument "{}" is not marked as mutable, but changed from {} to {}'.format(
argname, old_val, new_val)
debug('Mutablity problem:', detail)
return CallAnalysis(VerificationStatus.REFUTED, [
AnalysisMessage(MessageType.POST_ERR, detail, fn_filename,
fn_start_lineno, 0, '')
])
(post_condition, ) = conditions.post
with ExceptionFilter(expected_exceptions) as efilter:
# TODO: re-enable post-condition short circuiting. This will require refactoring how
# enforced conditions and short curcuiting interact, so that post-conditions are
# selectively run when, and only when, performing a short circuit.
#with enforced_conditions.enabled_enforcement(), short_circuit:
isok = bool(post_condition.evaluate(lcls))
if efilter.ignore:
debug('Ignored exception in postcondition.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
(e, tb) = efilter.user_exc
detail = repr(e) + ' ' + get_input_description(
space, fn.__name__, original_args, __return__,
post_condition.addl_context)
debug('exception while calling postcondition:', detail)
failures = [
AnalysisMessage(
MessageType.POST_ERR,
*locate_msg(detail, post_condition.filename,
post_condition.line), ''.join(tb.format()))
]
return CallAnalysis(VerificationStatus.REFUTED, failures)
if isok:
debug('Postcondition confirmed.')
return CallAnalysis(VerificationStatus.CONFIRMED)
else:
detail = 'false ' + \
get_input_description(
space, fn.__name__, original_args, __return__, post_condition.addl_context)
debug(detail)
failures = [
AnalysisMessage(
MessageType.POST_FAIL,
*locate_msg(detail, post_condition.filename,
post_condition.line), '')
]
return CallAnalysis(VerificationStatus.REFUTED, failures)