def run_iteration( fn1: Callable, fn2: Callable, sig: inspect.Signature, space: StateSpace ) -> Tuple[Optional[VerificationStatus], Optional[BehaviorDiff]]: original_args = gen_args(sig) args1 = copy.deepcopy(original_args) args2 = copy.deepcopy(original_args) coverage_manager = measure_fn_coverage(fn1, fn2) with ExceptionFilter() as efilter, coverage_manager as coverage: result1 = describe_behavior(fn1, args1) result2 = describe_behavior(fn2, args2) space.check_deferred_assumptions() if result1 == result2 and args1 == args2: debug("Functions equivalent") return (VerificationStatus.CONFIRMED, None) debug("Functions differ") realized_args = { k: repr(v) for (k, v) in original_args.arguments.items() } post_execution_args1 = {k: repr(v) for k, v in args1.arguments.items()} post_execution_args2 = {k: repr(v) for k, v in args2.arguments.items()} diff = BehaviorDiff( realized_args, Result(repr(result1[0]), result1[1], post_execution_args1), Result(repr(result2[0]), result2[1], post_execution_args2), coverage(fn1), coverage(fn2), ) return (VerificationStatus.REFUTED, diff) if efilter.user_exc: debug("User-level exception found", repr(efilter.user_exc[0]), efilter.user_exc[1]) return (None, None)
def run_iteration(fn: Callable, sig: Signature, space: StateSpace) -> Optional[PathSummary]: with NoTracing(): args = gen_args(sig) pre_args = copy.deepcopy(args) ret = None with measure_fn_coverage(fn) as coverage, ExceptionFilter() as efilter: # coverage = lambda _: CoverageResult(set(), set(), 1.0) # with ExceptionFilter() as efilter: ret = fn(*args.args, **args.kwargs) space.detach_path() if efilter.user_exc is not None: exc = efilter.user_exc[0] debug("user-level exception found", repr(exc), *efilter.user_exc[1]) return PathSummary(pre_args, ret, type(exc), args, coverage(fn)) elif efilter.ignore: return None else: return PathSummary( deep_realize(pre_args), deep_realize(ret), None, deep_realize(args), coverage(fn), )
def choose_type(space: StateSpace, from_type: Type) -> Type: subtypes = get_subclass_map()[from_type] # Note that this is written strangely to leverage the default # preference for false when forking: if not subtypes or not space.smt_fork(): return from_type for subtype in subtypes[:-1]: if not space.smt_fork(): return choose_type(space, subtype) return choose_type(space, subtypes[-1])
def path_cover(ctxfn: FunctionInfo, options: AnalysisOptions, coverage_type: CoverageType) -> List[PathSummary]: fn, sig = ctxfn.callable() search_root = SinglePathNode(True) condition_start = time.monotonic() paths: List[PathSummary] = [] for i in range(1, options.max_iterations): debug("Iteration ", i) itr_start = time.monotonic() if itr_start > condition_start + options.per_condition_timeout: debug( "Stopping due to --per_condition_timeout=", options.per_condition_timeout, ) break space = StateSpace( execution_deadline=itr_start + options.per_path_timeout, model_check_timeout=options.per_path_timeout / 2, search_root=search_root, ) with condition_parser(options.analysis_kind), Patched( ), COMPOSITE_TRACER, StateSpaceContext(space): summary = None try: summary = run_iteration(fn, sig, space) verification_status = VerificationStatus.CONFIRMED except UnexploredPath: verification_status = VerificationStatus.UNKNOWN debug("Verification status:", verification_status) top_analysis, exhausted = space.bubble_status( CallAnalysis(verification_status)) debug("Path tree stats", search_root.stats()) if summary: paths.append(summary) if exhausted: debug("Stopping due to code path exhaustion. (yay!)") break opcodes_found: Set[int] = set() selected: List[PathSummary] = [] while paths: next_best = max( paths, key=lambda p: len(p.coverage.offsets_covered - opcodes_found)) cur_offsets = next_best.coverage.offsets_covered if coverage_type == CoverageType.OPCODE: if len(cur_offsets - opcodes_found) == 0: break selected.append(next_best) opcodes_found |= cur_offsets paths = [p for p in paths if p is not next_best] return selected
def symbolic_run( self, fn: Callable[[StateSpace, Dict[str, object]], object], typed_args: Dict[str, type], ) -> Tuple[object, # return value Optional[Dict[str, object]], # arguments after execution Optional[BaseException], # exception thrown, if any StateSpace, ]: search_root = SinglePathNode(True) with COMPOSITE_TRACER, Patched(): for itr in range(1, 200): debug("iteration", itr) space = StateSpace(time.monotonic() + 10.0, 1.0, search_root=search_root) symbolic_args = {} try: with StateSpaceContext(space): symbolic_args = { name: proxy_for_type(typ, name) for name, typ in typed_args.items() } ret = fn(space, symbolic_args) ret = (deep_realize(ret), symbolic_args, None, space) space.check_deferred_assumptions() return ret except IgnoreAttempt as e: debug("ignore iteration attempt: ", str(e)) pass except BaseException as e: debug(traceback.format_exc()) return (None, symbolic_args, e, space) top_analysis, space_exhausted = space.bubble_status( CallAnalysis()) if space_exhausted: return ( None, symbolic_args, CrosshairInternal(f"exhausted after {itr} iterations"), space, ) return ( None, None, CrosshairInternal( "Unable to find a successful symbolic execution"), space, )
def proxy_for_type(typ: Type, space: StateSpace, varname: str, meet_class_invariants=True, allow_subtypes=False) -> object: typ = normalize_pytype(typ) origin = origin_of(typ) type_args = type_args_of(typ) # special cases if isinstance(typ, type) and issubclass(typ, enum.Enum): enum_values = list(typ) # type:ignore for enum_value in enum_values[:-1]: if space.smt_fork(): return enum_value return enum_values[-1] proxy_factory = _SIMPLE_PROXIES.get(origin) if proxy_factory: def recursive_proxy_factory(t: Type): return proxy_for_type(t, space, varname + space.uniq(), allow_subtypes=allow_subtypes) recursive_proxy_factory.space = space # type: ignore recursive_proxy_factory.pytype = typ # type: ignore recursive_proxy_factory.varname = varname # type: ignore return proxy_factory(recursive_proxy_factory, *type_args) if allow_subtypes and typ is not object: typ = choose_type(space, typ) return proxy_for_class(typ, space, varname, meet_class_invariants)
def pick_code(space: StateSpace) -> Tuple[str, int, int]: last_idx = len(INT_TYPE_BOUNDS) - 1 for (idx, (code, rng)) in enumerate(INT_TYPE_BOUNDS.items()): if idx < last_idx: if space.smt_fork(desc=f"not_{code}_array"): continue return (code, *rng) assert False, "Not Reachable"
def diff_behavior_with_signature( fn1: Callable, fn2: Callable, sig: inspect.Signature, options: AnalysisOptions ) -> Iterable[BehaviorDiff]: search_root = SinglePathNode(True) condition_start = time.monotonic() for i in range(1, options.max_iterations): debug("Iteration ", i) itr_start = time.monotonic() if itr_start > condition_start + options.per_condition_timeout: debug( "Stopping due to --per_condition_timeout=", options.per_condition_timeout, ) return options.incr("num_paths") space = StateSpace( execution_deadline=itr_start + options.per_path_timeout, model_check_timeout=options.per_path_timeout / 2, search_root=search_root, ) with StateSpaceContext(space): output = None try: (verification_status, output) = run_iteration(fn1, fn2, sig, space) except UnexploredPath: verification_status = VerificationStatus.UNKNOWN debug("Verification status:", verification_status) top_analysis, space_exhausted = space.bubble_status( CallAnalysis(verification_status) ) if ( top_analysis and top_analysis.verification_status == VerificationStatus.CONFIRMED ): debug("Stopping due to code path exhaustion. (yay!)") options.incr("exhaustion") break if output: yield output
def make_fake_object(statespace: StateSpace, cls: type, varname: str) -> object: constructor = get_smt_proxy_type(cls) debug(constructor) try: proxy = constructor() except TypeError as e: # likely the type has a __new__ that expects arguments raise CrosshairUnsupported(f'Unable to proxy {name_of_type(cls)}: {e}') for name, typ in get_type_hints(cls).items(): origin = getattr(typ, '__origin__', None) if origin is Callable: continue value = proxy_for_type(typ, statespace, varname + '.' + name + statespace.uniq()) object.__setattr__(proxy, name, value) return proxy
def gen_args(sig: inspect.Signature, statespace: StateSpace) -> inspect.BoundArguments: args = sig.bind_partial() for param in sig.parameters.values(): smt_name = param.name + statespace.uniq() proxy_maker = lambda typ, **kw: proxy_for_type( typ, statespace, smt_name, allow_subtypes=True, **kw) has_annotation = (param.annotation != inspect.Parameter.empty) value: object if param.kind == inspect.Parameter.VAR_POSITIONAL: if has_annotation: varargs_type = List[param.annotation] # type: ignore value = proxy_maker(varargs_type) else: value = proxy_maker(List[Any]) elif param.kind == inspect.Parameter.VAR_KEYWORD: if has_annotation: varargs_type = Dict[str, param.annotation] # type: ignore value = cast(dict, proxy_maker(varargs_type)) # Using ** on a dict requires concrete string keys. Force # instiantiation of keys here: value = {k.__str__(): v for (k, v) in value.items()} else: value = proxy_maker(Dict[str, Any]) else: is_self = param.name == 'self' # Object parameters should meet thier invariants iff they are not the # class under test ("self"). meet_class_invariants = not is_self allow_subtypes = not is_self if has_annotation: value = proxy_for_type(param.annotation, statespace, smt_name, meet_class_invariants, allow_subtypes) else: value = proxy_for_type(cast(type, Any), statespace, smt_name, meet_class_invariants, allow_subtypes) debug('created proxy for', param.name, 'as type:', type(value)) args.arguments[param.name] = value return args
def __ch_forget_contents__(self, space: StateSpace): cls = self.__ch_pytype__() clean = proxy_for_type(cls, space, space.uniq()) for name, val in self.__dict__.items(): self.__dict__[name] = clean.__dict__[name]
def attempt_call(conditions: Conditions, space: StateSpace, fn: Callable, short_circuit: ShortCircuitingContext, enforced_conditions: EnforcedConditions) -> CallAnalysis: bound_args = gen_args(conditions.sig, space) code_obj = fn.__code__ fn_filename, fn_start_lineno = (code_obj.co_filename, code_obj.co_firstlineno) try: (lines, _) = inspect.getsourcelines(fn) except OSError: lines = [] fn_end_lineno = fn_start_lineno + len(lines) def locate_msg(detail: str, suggested_filename: str, suggested_lineno: int) -> Tuple[str, str, int, int]: if ((os.path.abspath(suggested_filename) == os.path.abspath(fn_filename)) and (fn_start_lineno <= suggested_lineno <= fn_end_lineno)): return (detail, suggested_filename, suggested_lineno, 0) else: try: exprline = linecache.getlines(suggested_filename)[ suggested_lineno - 1].strip() except IndexError: exprline = '<unknown>' detail = f'"{exprline}" yields {detail}' return (detail, fn_filename, fn_start_lineno, 0) with space.framework(): original_args = copy.deepcopy(bound_args) space.checkpoint() lcls: Mapping[str, object] = bound_args.arguments # In preconditions, __old__ exists but is just bound to the same args. # This lets people write class invariants using `__old__` to, for example, # demonstrate immutability. lcls = {'__old__': AttributeHolder(lcls), **lcls} expected_exceptions = conditions.raises for precondition in conditions.pre: with ExceptionFilter(expected_exceptions) as efilter: with enforced_conditions.enabled_enforcement(), short_circuit: precondition_ok = precondition.evaluate(lcls) if not precondition_ok: debug('Failed to meet precondition', precondition.expr_source) return CallAnalysis(failing_precondition=precondition) if efilter.ignore: debug('Ignored exception in precondition.', efilter.analysis) return efilter.analysis elif efilter.user_exc is not None: (user_exc, tb) = efilter.user_exc debug('Exception attempting to meet precondition', precondition.expr_source, ':', user_exc, tb.format()) return CallAnalysis( failing_precondition=precondition, failing_precondition_reason= f'it raised "{repr(user_exc)} at {tb.format()[-1]}"') with ExceptionFilter(expected_exceptions) as efilter: a, kw = bound_args.args, bound_args.kwargs with enforced_conditions.enabled_enforcement(), short_circuit: assert not space.running_framework_code __return__ = fn(*a, **kw) lcls = { **bound_args.arguments, '__return__': __return__, '_': __return__, '__old__': AttributeHolder(original_args.arguments), fn.__name__: fn } if efilter.ignore: debug('Ignored exception in function.', efilter.analysis) return efilter.analysis elif efilter.user_exc is not None: (e, tb) = efilter.user_exc detail = name_of_type(type(e)) + ': ' + str(e) frame_filename, frame_lineno = frame_summary_for_fn(tb, fn) debug('exception while evaluating function body:', detail, frame_filename, 'line', frame_lineno) detail += ' ' + get_input_description(space, fn.__name__, original_args, _MISSING) return CallAnalysis(VerificationStatus.REFUTED, [ AnalysisMessage(MessageType.EXEC_ERR, *locate_msg(detail, frame_filename, frame_lineno), ''.join(tb.format())) ]) for argname, argval in bound_args.arguments.items(): if (conditions.mutable_args is not None and argname not in conditions.mutable_args): old_val, new_val = original_args.arguments[argname], argval if not deep_eq(old_val, new_val, set()): detail = 'Argument "{}" is not marked as mutable, but changed from {} to {}'.format( argname, old_val, new_val) debug('Mutablity problem:', detail) return CallAnalysis(VerificationStatus.REFUTED, [ AnalysisMessage(MessageType.POST_ERR, detail, fn_filename, fn_start_lineno, 0, '') ]) (post_condition, ) = conditions.post with ExceptionFilter(expected_exceptions) as efilter: # TODO: re-enable post-condition short circuiting. This will require refactoring how # enforced conditions and short curcuiting interact, so that post-conditions are # selectively run when, and only when, performing a short circuit. #with enforced_conditions.enabled_enforcement(), short_circuit: isok = bool(post_condition.evaluate(lcls)) if efilter.ignore: debug('Ignored exception in postcondition.', efilter.analysis) return efilter.analysis elif efilter.user_exc is not None: (e, tb) = efilter.user_exc detail = repr(e) + ' ' + get_input_description( space, fn.__name__, original_args, __return__, post_condition.addl_context) debug('exception while calling postcondition:', detail) failures = [ AnalysisMessage( MessageType.POST_ERR, *locate_msg(detail, post_condition.filename, post_condition.line), ''.join(tb.format())) ] return CallAnalysis(VerificationStatus.REFUTED, failures) if isok: debug('Postcondition confirmed.') return CallAnalysis(VerificationStatus.CONFIRMED) else: detail = 'false ' + \ get_input_description( space, fn.__name__, original_args, __return__, post_condition.addl_context) debug(detail) failures = [ AnalysisMessage( MessageType.POST_FAIL, *locate_msg(detail, post_condition.filename, post_condition.line), '') ] return CallAnalysis(VerificationStatus.REFUTED, failures)