def symbolic_run(
self, fn: Callable[[TrackingStateSpace], object]
) -> Tuple[object, Optional[BaseException], TrackingStateSpace]:
search_root = SinglePathNode(True)
with Patched(enabled=lambda: True):
for itr in range(1, 200):
debug('iteration', itr)
space = TrackingStateSpace(time.monotonic() + 10.0,
1.0,
search_root=search_root)
try:
with StateSpaceContext(space):
ret = (realize(fn(space)), None, space)
space.check_deferred_assumptions()
return ret
except IgnoreAttempt as e:
debug('ignore iteration attempt: ', str(e))
pass
except BaseException as e:
debug(traceback.format_exc())
return (None, e, space)
top_analysis, space_exhausted = space.bubble_status(
CallAnalysis())
if space_exhausted:
return (
None,
CrosshairInternal(f'exhausted after {itr} iterations'),
space)
return (None,
CrosshairInternal(
'Unable to find a successful symbolic execution'), space)
def symbolic_run(
self, fn: Callable[[TrackingStateSpace], bool]
) -> Tuple[object, Optional[BaseException]]:
search_root = SinglePathNode(True)
patched_builtins = PatchedBuiltins(contracted_builtins.__dict__,
enabled=lambda: True)
with patched_builtins:
for itr in range(1, 200):
debug('iteration', itr)
space = TrackingStateSpace(time.time() + 10.0,
1.0,
search_root=search_root)
try:
return (realize(fn(space)), None)
except IgnoreAttempt as e:
debug('ignore iteration attempt: ', str(e))
pass
except BaseException as e:
#traceback.print_exc()
return (None, e)
top_analysis, space_exhausted = space.bubble_status(
CallAnalysis())
if space_exhausted:
return (
None,
CrosshairInternal(f'exhausted after {itr} iterations'))
return (None,
CrosshairInternal(
'Unable to find a successful symbolic execution'))
def symbolic_run(
self, fn: Callable[[TrackingStateSpace], bool]
) -> Tuple[object, Optional[BaseException]]:
search_root = SinglePathNode(True)
itr = 0
for _ in range(200):
itr += 1
space = TrackingStateSpace(time.time() + 10.0,
1.0,
search_root=search_root)
try:
return (realize(fn(space)), None)
except IgnoreAttempt:
pass
except BaseException as e:
#import traceback
#traceback.print_exc()
return (None, e)
top_analysis, space_exhausted = space.bubble_status(CallAnalysis())
if space_exhausted:
return (None,
CrosshairInternal(f'exhausted after {itr} iterations'))
return (None,
CrosshairInternal(
'Unable to find a successful symbolic execution'))
def __exit__(self, exc_type, exc_value, tb):
if isinstance(exc_value, (PostconditionFailed, IgnoreAttempt)):
if isinstance(exc_value, PostconditionFailed):
# Postcondition : although this indicates a problem, it's with a
# subroutine; not this function.
# Usualy we want to ignore this because it will be surfaced more locally
# in the subroutine.
debug(
f'Ignoring based on internal failed post condition: {exc_value}'
)
self.ignore = True
self.analysis = CallAnalysis()
return True
if isinstance(exc_value, self.expected_exceptions):
debug(f'Hit expected exception: {exc_value}')
self.ignore = True
self.analysis = CallAnalysis(VerificationStatus.CONFIRMED)
return True
if isinstance(exc_value, TypeError):
exc_str = str(exc_value)
if ('SmtStr' in exc_str or 'SmtInt' in exc_str
or 'SmtFloat' in exc_str
or 'expected string or bytes-like object' in exc_str):
# Ideally we'd attempt literal strings after encountering this.
# See https://github.com/pschanely/CrossHair/issues/8
debug('Proxy intolerace at: ', traceback.format_exc())
raise CrosshairUnsupported('Detected proxy intolerance: ' +
exc_str)
if isinstance(exc_value,
(UnexploredPath, CrosshairInternal, z3.Z3Exception)):
return False # internal issue: re-raise
if isinstance(
exc_value,
BaseException): # TODO: should this be "Exception" instead?
# Most other issues are assumed to be user-level exceptions:
self.user_exc = (exc_value,
traceback.extract_tb(sys.exc_info()[2]))
self.analysis = CallAnalysis(VerificationStatus.REFUTED)
return True # suppress user-level exception
return False # re-raise resource and system issues
def path_cover(ctxfn: FunctionInfo, options: AnalysisOptions,
coverage_type: CoverageType) -> List[PathSummary]:
fn, sig = ctxfn.callable()
search_root = SinglePathNode(True)
condition_start = time.monotonic()
paths: List[PathSummary] = []
for i in range(1, options.max_iterations):
debug("Iteration ", i)
itr_start = time.monotonic()
if itr_start > condition_start + options.per_condition_timeout:
debug(
"Stopping due to --per_condition_timeout=",
options.per_condition_timeout,
)
break
space = StateSpace(
execution_deadline=itr_start + options.per_path_timeout,
model_check_timeout=options.per_path_timeout / 2,
search_root=search_root,
)
with condition_parser(options.analysis_kind), Patched(
), COMPOSITE_TRACER, StateSpaceContext(space):
summary = None
try:
summary = run_iteration(fn, sig, space)
verification_status = VerificationStatus.CONFIRMED
except UnexploredPath:
verification_status = VerificationStatus.UNKNOWN
debug("Verification status:", verification_status)
top_analysis, exhausted = space.bubble_status(
CallAnalysis(verification_status))
debug("Path tree stats", search_root.stats())
if summary:
paths.append(summary)
if exhausted:
debug("Stopping due to code path exhaustion. (yay!)")
break
opcodes_found: Set[int] = set()
selected: List[PathSummary] = []
while paths:
next_best = max(
paths,
key=lambda p: len(p.coverage.offsets_covered - opcodes_found))
cur_offsets = next_best.coverage.offsets_covered
if coverage_type == CoverageType.OPCODE:
if len(cur_offsets - opcodes_found) == 0:
break
selected.append(next_best)
opcodes_found |= cur_offsets
paths = [p for p in paths if p is not next_best]
return selected
def symbolic_run(
self,
fn: Callable[[StateSpace, Dict[str, object]], object],
typed_args: Dict[str, type],
) -> Tuple[object, # return value
Optional[Dict[str, object]], # arguments after execution
Optional[BaseException], # exception thrown, if any
StateSpace, ]:
search_root = SinglePathNode(True)
with COMPOSITE_TRACER, Patched():
for itr in range(1, 200):
debug("iteration", itr)
space = StateSpace(time.monotonic() + 10.0,
1.0,
search_root=search_root)
symbolic_args = {}
try:
with StateSpaceContext(space):
symbolic_args = {
name: proxy_for_type(typ, name)
for name, typ in typed_args.items()
}
ret = fn(space, symbolic_args)
ret = (deep_realize(ret), symbolic_args, None, space)
space.check_deferred_assumptions()
return ret
except IgnoreAttempt as e:
debug("ignore iteration attempt: ", str(e))
pass
except BaseException as e:
debug(traceback.format_exc())
return (None, symbolic_args, e, space)
top_analysis, space_exhausted = space.bubble_status(
CallAnalysis())
if space_exhausted:
return (
None,
symbolic_args,
CrosshairInternal(f"exhausted after {itr} iterations"),
space,
)
return (
None,
None,
CrosshairInternal(
"Unable to find a successful symbolic execution"),
space,
)
def diff_behavior_with_signature(
fn1: Callable, fn2: Callable, sig: inspect.Signature, options: AnalysisOptions) -> Iterable[str]:
search_root = SinglePathNode(True)
condition_start = time.monotonic()
for i in range(1, options.max_iterations):
debug('Iteration ', i)
itr_start = time.monotonic()
if itr_start > condition_start + options.per_condition_timeout:
yield f'(stopping due to --per_condition_timeout={options.per_condition_timeout})'
break
space = TrackingStateSpace(
execution_deadline=itr_start + options.per_path_timeout,
model_check_timeout=options.per_path_timeout / 2,
search_root=search_root)
with StateSpaceContext(space):
(verification_status, output) = run_iteration(fn1, fn2, sig, space)
debug('Verification status:', verification_status)
top_analysis, space_exhausted = space.bubble_status(CallAnalysis(verification_status))
if top_analysis and top_analysis.verification_status == VerificationStatus.CONFIRMED:
break
yield from output
def diff_behavior_with_signature(
fn1: Callable, fn2: Callable, sig: inspect.Signature, options: AnalysisOptions
) -> Iterable[BehaviorDiff]:
search_root = SinglePathNode(True)
condition_start = time.monotonic()
for i in range(1, options.max_iterations):
debug("Iteration ", i)
itr_start = time.monotonic()
if itr_start > condition_start + options.per_condition_timeout:
debug(
"Stopping due to --per_condition_timeout=",
options.per_condition_timeout,
)
return
options.incr("num_paths")
space = StateSpace(
execution_deadline=itr_start + options.per_path_timeout,
model_check_timeout=options.per_path_timeout / 2,
search_root=search_root,
)
with StateSpaceContext(space):
output = None
try:
(verification_status, output) = run_iteration(fn1, fn2, sig, space)
except UnexploredPath:
verification_status = VerificationStatus.UNKNOWN
debug("Verification status:", verification_status)
top_analysis, space_exhausted = space.bubble_status(
CallAnalysis(verification_status)
)
if (
top_analysis
and top_analysis.verification_status == VerificationStatus.CONFIRMED
):
debug("Stopping due to code path exhaustion. (yay!)")
options.incr("exhaustion")
break
if output:
yield output
def analyze_calltree(options: AnalysisOptions,
conditions: Conditions) -> CallTreeAnalysis:
fn = conditions.fn
debug('Begin analyze calltree ', fn.__name__)
all_messages = MessageCollector()
search_root = SinglePathNode(True)
space_exhausted = False
failing_precondition: Optional[
ConditionExpr] = conditions.pre[0] if conditions.pre else None
failing_precondition_reason: str = ''
num_confirmed_paths = 0
_ = get_subclass_map() # ensure loaded
short_circuit = ShortCircuitingContext()
top_analysis: Optional[CallAnalysis] = None
enforced_conditions = EnforcedConditions(
options.condition_parser(),
fn_globals(fn),
builtin_patches(),
interceptor=short_circuit.make_interceptor)
def in_symbolic_mode():
space = optional_context_statespace()
return space and not space.running_framework_code
patched = Patched(in_symbolic_mode)
with enforced_conditions, patched, enforced_conditions.disabled_enforcement(
):
for i in itertools.count(1):
start = time.monotonic()
if start > options.deadline:
debug('Exceeded condition timeout, stopping')
break
options.incr('num_paths')
debug('Iteration ', i)
space = TrackingStateSpace(
execution_deadline=start + options.per_path_timeout,
model_check_timeout=options.per_path_timeout / 2,
search_root=search_root)
try:
# The real work happens here!:
with StateSpaceContext(space):
call_analysis = attempt_call(conditions, fn, short_circuit,
enforced_conditions)
if failing_precondition is not None:
cur_precondition = call_analysis.failing_precondition
if cur_precondition is None:
if call_analysis.verification_status is not None:
# We escaped the all the pre conditions on this try:
failing_precondition = None
elif (cur_precondition.line == failing_precondition.line
and call_analysis.failing_precondition_reason):
failing_precondition_reason = call_analysis.failing_precondition_reason
elif cur_precondition.line > failing_precondition.line:
failing_precondition = cur_precondition
failing_precondition_reason = call_analysis.failing_precondition_reason
except UnexploredPath:
call_analysis = CallAnalysis(VerificationStatus.UNKNOWN)
except IgnoreAttempt:
call_analysis = CallAnalysis()
status = call_analysis.verification_status
if status == VerificationStatus.CONFIRMED:
num_confirmed_paths += 1
top_analysis, space_exhausted = space.bubble_status(call_analysis)
overall_status = top_analysis.verification_status if top_analysis else None
debug('Iter complete. Worst status found so far:',
overall_status.name if overall_status else 'None')
if space_exhausted or top_analysis == VerificationStatus.REFUTED:
break
top_analysis = search_root.child.get_result()
if top_analysis.messages:
#log = space.execution_log()
all_messages.extend(
replace(
m,
#execution_log=log,
test_fn=fn.__qualname__,
condition_src=conditions.post[0].expr_source)
for m in top_analysis.messages)
if top_analysis.verification_status is None:
top_analysis.verification_status = VerificationStatus.UNKNOWN
if failing_precondition:
assert num_confirmed_paths == 0
addl_ctx = ' ' + failing_precondition.addl_context if failing_precondition.addl_context else ''
message = f'Unable to meet precondition{addl_ctx}'
if failing_precondition_reason:
message += f' (possibly because {failing_precondition_reason}?)'
all_messages.extend([
AnalysisMessage(MessageType.PRE_UNSAT, message + '.',
failing_precondition.filename,
failing_precondition.line, 0, '')
])
top_analysis = CallAnalysis(VerificationStatus.REFUTED)
assert top_analysis.verification_status is not None
debug(
('Exhausted' if space_exhausted else 'Aborted'),
' calltree search with', top_analysis.verification_status.name, 'and',
len(all_messages.get()), 'messages.', 'Number of iterations: ', i)
return CallTreeAnalysis(
messages=all_messages.get(),
verification_status=top_analysis.verification_status,
num_confirmed_paths=num_confirmed_paths)
def attempt_call(conditions: Conditions, fn: Callable,
short_circuit: ShortCircuitingContext,
enforced_conditions: EnforcedConditions) -> CallAnalysis:
space = context_statespace()
bound_args = gen_args(conditions.sig)
msg_gen = MessageGenerator(fn)
with space.framework():
# TODO: looks wrong(-ish) to guard this with space.framework().
# Copy on custom objects may require patched builtins. (datetime.timedelta is one such case)
original_args = copy.deepcopy(bound_args)
space.checkpoint()
lcls: Mapping[str, object] = bound_args.arguments
# In preconditions, __old__ exists but is just bound to the same args.
# This lets people write class invariants using `__old__` to, for example,
# demonstrate immutability.
lcls = {'__old__': AttributeHolder(lcls), **lcls}
expected_exceptions = conditions.raises
for precondition in conditions.pre:
with ExceptionFilter(expected_exceptions) as efilter:
with enforced_conditions.enabled_enforcement(), short_circuit:
precondition_ok = precondition.evaluate(lcls)
if not precondition_ok:
debug('Failed to meet precondition', precondition.expr_source)
return CallAnalysis(failing_precondition=precondition)
if efilter.ignore:
debug('Ignored exception in precondition.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
(user_exc, tb) = efilter.user_exc
debug('Exception attempting to meet precondition',
precondition.expr_source, ':', user_exc, tb.format())
return CallAnalysis(
failing_precondition=precondition,
failing_precondition_reason=
f'it raised "{repr(user_exc)} at {tb.format()[-1]}"')
with ExceptionFilter(expected_exceptions) as efilter:
with enforced_conditions.enabled_enforcement(), short_circuit:
assert not space.running_framework_code
__return__ = fn(*bound_args.args, **bound_args.kwargs)
lcls = {
**bound_args.arguments, '__return__': __return__,
'_': __return__,
'__old__': AttributeHolder(original_args.arguments),
fn.__name__: fn
}
if efilter.ignore:
debug('Ignored exception in function.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
space.check_deferred_assumptions()
(e, tb) = efilter.user_exc
detail = name_of_type(type(e)) + ': ' + str(e)
frame_filename, frame_lineno = frame_summary_for_fn(tb, fn)
debug('exception while evaluating function body:', detail,
frame_filename, 'line', frame_lineno)
detail += ' ' + get_input_description(fn.__name__, original_args,
_MISSING)
return CallAnalysis(VerificationStatus.REFUTED, [
msg_gen.make(MessageType.EXEC_ERR, detail, frame_filename,
frame_lineno, ''.join(tb.format()))
])
for argname, argval in bound_args.arguments.items():
if (conditions.mutable_args is not None
and argname not in conditions.mutable_args):
old_val, new_val = original_args.arguments[argname], argval
if not deep_eq(old_val, new_val, set()):
space.check_deferred_assumptions()
detail = 'Argument "{}" is not marked as mutable, but changed from {} to {}'.format(
argname, old_val, new_val)
debug('Mutablity problem:', detail)
return CallAnalysis(
VerificationStatus.REFUTED,
[msg_gen.make(MessageType.POST_ERR, detail, None, 0, '')])
(post_condition, ) = conditions.post
with ExceptionFilter(expected_exceptions) as efilter:
# TODO: re-enable post-condition short circuiting. This will require refactoring how
# enforced conditions and short curcuiting interact, so that post-conditions are
# selectively run when, and only when, performing a short circuit.
#with enforced_conditions.enabled_enforcement(), short_circuit:
isok = bool(post_condition.evaluate(lcls))
if efilter.ignore:
debug('Ignored exception in postcondition.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
space.check_deferred_assumptions()
(e, tb) = efilter.user_exc
detail = repr(e) + ' ' + get_input_description(
fn.__name__, original_args, __return__,
post_condition.addl_context)
debug('exception while calling postcondition:', detail)
failures = [
msg_gen.make(MessageType.POST_ERR, detail, post_condition.filename,
post_condition.line, ''.join(tb.format()))
]
return CallAnalysis(VerificationStatus.REFUTED, failures)
if isok:
debug('Postcondition confirmed.')
return CallAnalysis(VerificationStatus.CONFIRMED)
else:
space.check_deferred_assumptions()
detail = 'false ' + \
get_input_description(
fn.__name__, original_args, __return__, post_condition.addl_context)
debug(detail)
failures = [
msg_gen.make(MessageType.POST_FAIL, detail,
post_condition.filename, post_condition.line, '')
]
return CallAnalysis(VerificationStatus.REFUTED, failures)
def attempt_call(conditions: Conditions, space: StateSpace, fn: Callable,
short_circuit: ShortCircuitingContext,
enforced_conditions: EnforcedConditions) -> CallAnalysis:
bound_args = gen_args(conditions.sig, space)
code_obj = fn.__code__
fn_filename, fn_start_lineno = (code_obj.co_filename,
code_obj.co_firstlineno)
try:
(lines, _) = inspect.getsourcelines(fn)
except OSError:
lines = []
fn_end_lineno = fn_start_lineno + len(lines)
def locate_msg(detail: str, suggested_filename: str,
suggested_lineno: int) -> Tuple[str, str, int, int]:
if ((os.path.abspath(suggested_filename)
== os.path.abspath(fn_filename))
and (fn_start_lineno <= suggested_lineno <= fn_end_lineno)):
return (detail, suggested_filename, suggested_lineno, 0)
else:
try:
exprline = linecache.getlines(suggested_filename)[
suggested_lineno - 1].strip()
except IndexError:
exprline = '<unknown>'
detail = f'"{exprline}" yields {detail}'
return (detail, fn_filename, fn_start_lineno, 0)
with space.framework():
original_args = copy.deepcopy(bound_args)
space.checkpoint()
lcls: Mapping[str, object] = bound_args.arguments
# In preconditions, __old__ exists but is just bound to the same args.
# This lets people write class invariants using `__old__` to, for example,
# demonstrate immutability.
lcls = {'__old__': AttributeHolder(lcls), **lcls}
expected_exceptions = conditions.raises
for precondition in conditions.pre:
with ExceptionFilter(expected_exceptions) as efilter:
with enforced_conditions.enabled_enforcement(), short_circuit:
precondition_ok = precondition.evaluate(lcls)
if not precondition_ok:
debug('Failed to meet precondition', precondition.expr_source)
return CallAnalysis(failing_precondition=precondition)
if efilter.ignore:
debug('Ignored exception in precondition.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
(user_exc, tb) = efilter.user_exc
debug('Exception attempting to meet precondition',
precondition.expr_source, ':', user_exc, tb.format())
return CallAnalysis(
failing_precondition=precondition,
failing_precondition_reason=
f'it raised "{repr(user_exc)} at {tb.format()[-1]}"')
with ExceptionFilter(expected_exceptions) as efilter:
a, kw = bound_args.args, bound_args.kwargs
with enforced_conditions.enabled_enforcement(), short_circuit:
assert not space.running_framework_code
__return__ = fn(*a, **kw)
lcls = {
**bound_args.arguments, '__return__': __return__,
'_': __return__,
'__old__': AttributeHolder(original_args.arguments),
fn.__name__: fn
}
if efilter.ignore:
debug('Ignored exception in function.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
(e, tb) = efilter.user_exc
detail = name_of_type(type(e)) + ': ' + str(e)
frame_filename, frame_lineno = frame_summary_for_fn(tb, fn)
debug('exception while evaluating function body:', detail,
frame_filename, 'line', frame_lineno)
detail += ' ' + get_input_description(space, fn.__name__,
original_args, _MISSING)
return CallAnalysis(VerificationStatus.REFUTED, [
AnalysisMessage(MessageType.EXEC_ERR,
*locate_msg(detail, frame_filename, frame_lineno),
''.join(tb.format()))
])
for argname, argval in bound_args.arguments.items():
if (conditions.mutable_args is not None
and argname not in conditions.mutable_args):
old_val, new_val = original_args.arguments[argname], argval
if not deep_eq(old_val, new_val, set()):
detail = 'Argument "{}" is not marked as mutable, but changed from {} to {}'.format(
argname, old_val, new_val)
debug('Mutablity problem:', detail)
return CallAnalysis(VerificationStatus.REFUTED, [
AnalysisMessage(MessageType.POST_ERR, detail, fn_filename,
fn_start_lineno, 0, '')
])
(post_condition, ) = conditions.post
with ExceptionFilter(expected_exceptions) as efilter:
# TODO: re-enable post-condition short circuiting. This will require refactoring how
# enforced conditions and short curcuiting interact, so that post-conditions are
# selectively run when, and only when, performing a short circuit.
#with enforced_conditions.enabled_enforcement(), short_circuit:
isok = bool(post_condition.evaluate(lcls))
if efilter.ignore:
debug('Ignored exception in postcondition.', efilter.analysis)
return efilter.analysis
elif efilter.user_exc is not None:
(e, tb) = efilter.user_exc
detail = repr(e) + ' ' + get_input_description(
space, fn.__name__, original_args, __return__,
post_condition.addl_context)
debug('exception while calling postcondition:', detail)
failures = [
AnalysisMessage(
MessageType.POST_ERR,
*locate_msg(detail, post_condition.filename,
post_condition.line), ''.join(tb.format()))
]
return CallAnalysis(VerificationStatus.REFUTED, failures)
if isok:
debug('Postcondition confirmed.')
return CallAnalysis(VerificationStatus.CONFIRMED)
else:
detail = 'false ' + \
get_input_description(
space, fn.__name__, original_args, __return__, post_condition.addl_context)
debug(detail)
failures = [
AnalysisMessage(
MessageType.POST_FAIL,
*locate_msg(detail, post_condition.filename,
post_condition.line), '')
]
return CallAnalysis(VerificationStatus.REFUTED, failures)
