def get_class_conditions(self, cls: type) -> ClassConditions:
global_parser = self._global_parser
methods = {}
method_names = set(cls.__dict__.keys())
for method_name in method_names:
method = cls.__dict__.get(method_name, None)
if inspect.isfunction(method):
parsed_conditions = global_parser.get_fn_conditions(
method, first_arg_type=cls)
elif isinstance(method, classmethod):
method = method.__get__(cls).__func__ # type: ignore
parsed_conditions = global_parser.get_fn_conditions(
method, first_arg_type=type(cls))
elif isinstance(method, staticmethod):
parsed_conditions = global_parser.get_fn_conditions(
method.__get__(cls), first_arg_type=None)
else:
#debug('Skipping unhandled member type ', type(method), ': ', method_name)
continue
if parsed_conditions is None:
debug('Skipping ', str(method),
': Unable to determine the function signature.')
continue
if parsed_conditions.has_any():
methods[method_name] = parsed_conditions
return ClassConditions([], methods)
def __init__(self, rand: random.Random, expr: z3.ExprRef, solver: z3.Solver):
if self.condition_value is None:
if not solver_is_sat(solver):
debug('bad solver', solver.sexpr())
raise CrosshairInternal('unexpected un sat')
self.condition_value = solver.model().evaluate(expr, model_completion=True)
WorstResultNode.__init__(self, rand, expr == self.condition_value, solver)
def run_class_method_trials(self, cls: Type, min_trials: int) -> None:
debug('Checking class', cls)
for method_name, method in list(inspect.getmembers(cls)):
# We expect some methods to be different (at least, for now):
if method_name.startswith('__'):
continue
if method_name.startswith(
'_c_'): # Leftovers from forbiddenfruit curses
continue
if not (inspect.isfunction(method)
or inspect.ismethoddescriptor(method)):
continue
sig, _err = resolve_signature(method)
if sig is None:
continue
debug('Checking method', method_name)
num_trials = min_trials # TODO: something like this?: min_trials + round(len(sig.parameters) ** 1.5)
arg_names = [
chr(ord('a') + i - 1) for i in range(1, len(sig.parameters))
]
# TODO: some methods take kw-only args (list.sort for example):
expr_str = 'self.' + method_name + '(' + ','.join(arg_names) + ')'
arg_type_roots = {name: object for name in arg_names}
arg_type_roots['self'] = cls
num_unsupported = 0
for trial_num in range(num_trials):
status = self.run_trial(expr_str, arg_type_roots,
f'{method_name} #{trial_num}')
if status is TrialStatus.UNSUPPORTED:
num_unsupported += 1
if num_unsupported == num_trials:
self.fail(
f'{num_unsupported} unsupported cases out of {num_trials} testing the method "{method_name}"'
)
def merge_fn_conditions(sub_conditions: Conditions,
super_conditions: Conditions) -> Conditions:
# TODO: resolve the warning below:
# (1) the type of self always changes
# (2) paramter renames (or *a, **kws) could result in varied bindings
if sub_conditions.sig is not None and sub_conditions.sig != super_conditions.sig:
debug("WARNING: inconsistent signatures", sub_conditions.sig,
super_conditions.sig)
pre = sub_conditions.pre if sub_conditions.pre else super_conditions.pre
post = super_conditions.post + sub_conditions.post
raises = sub_conditions.raises | super_conditions.raises
mutable_args = (sub_conditions.mutable_args if sub_conditions.mutable_args
is not None else super_conditions.mutable_args)
return Conditions(
sub_conditions.fn,
sub_conditions.fn,
pre,
post,
raises,
sub_conditions.sig,
mutable_args,
sub_conditions.fn_syntax_messages,
)
def find_key_in_heap(
self,
ref: z3.ExprRef,
typ: Type,
proxy_generator: Callable[[Type], object],
snapshot: SnapshotRef = SnapshotRef(-1)
) -> object:
with self.framework():
for (curref, curtyp,
curval) in itertools.chain(*self.heaps[snapshot:]):
could_match = dynamic_typing.unify(curtyp, typ)
if not could_match:
continue
if self.smt_fork(curref == ref):
debug('HEAP key lookup ', ref, ': Found existing. ',
'type:', name_of_type(type(curval)), 'id:',
id(curval) % 1000)
return curval
ret = proxy_generator(typ)
debug('HEAP key lookup ', ref, ': Created new. ', 'type:',
name_of_type(type(ret)), 'id:',
id(ret) % 1000)
#assert dynamic_typing.unify(python_type(ret), typ), 'proxy type was {} and type required was {}'.format(type(ret), typ)
self.add_value_to_heaps(ref, typ, ret)
return ret
def test_example_coverage(self) -> None:
# Try to get examples that highlist the differences in the code.
# Here, we add more conditions for the `return True` path and
# another case where we used to just `return False`.
def isack1(s: str) -> bool:
if s in ("y", "yes"):
return True
return False
def isack2(s: str) -> Optional[bool]:
if s in ("y", "yes", "Y", "YES"):
return True
if s in ("n", "no", "N", "NO"):
return False
return None
diffs = diff_behavior(
FunctionInfo.from_fn(isack1),
FunctionInfo.from_fn(isack2),
DEFAULT_OPTIONS.overlay(max_iterations=20,
per_condition_timeout=5),
)
debug("diffs=", diffs)
assert not isinstance(diffs, str)
return_vals = set(
(d.result1.return_repr, d.result2.return_repr) for d in diffs)
self.assertEqual(return_vals, {("False", "None"), ("False", "True")})
def check(args: argparse.Namespace, options: AnalysisOptionSet, stdout: TextIO,
stderr: TextIO) -> int:
any_problems = False
try:
entities = list(load_files_or_qualnames(args.target))
except FileNotFoundError as exc:
print(f'File not found: "{exc.args[0]}"', file=stderr)
return 2
except ErrorDuringImport as exc:
cause = exc.__cause__ if exc.__cause__ is not None else exc
print(f"Could not import your code:\n", file=stderr)
traceback.print_exception(type(cause),
cause,
cause.__traceback__,
file=stderr)
return 2
full_options = DEFAULT_OPTIONS.overlay(
report_verbose=False).overlay(options)
for entity in entities:
debug("Check ", getattr(entity, "__name__", str(entity)))
for message in run_checkables(analyze_any(entity, options)):
line = describe_message(message, full_options)
if line is None:
continue
stdout.write(line + "\n")
debug("Traceback for output message:\n", message.traceback)
if message.state > MessageType.PRE_UNSAT:
any_problems = True
return 1 if any_problems else 0
def analyzable_filename(filename: str) -> bool:
"""
Check whether the file can be analyzed purely based on the ``filename``.
>>> analyzable_filename('foo23.py')
True
>>> analyzable_filename('#foo.py')
False
>>> analyzable_filename('23foo.py')
False
>>> analyzable_filename('setup.py')
False
"""
if not filename.endswith(".py"):
return False
lead_char = filename[0]
if (not lead_char.isalpha()) and (not lead_char.isidentifier()):
# (skip temporary editor files, backups, etc)
debug(
f"Skipping {filename} because it begins with a special character.")
return False
if filename in ("setup.py", ):
debug(
f"Skipping {filename} because files with this name are not usually import-able."
)
return False
return True
def proxy_for_class(typ: Type, varname: str,
meet_class_invariants: bool) -> object:
# if the class has data members, we attempt to create a concrete instance with
# symbolic members; otherwise, we'll create an object proxy that emulates it.
obj = proxy_class_as_concrete(typ, varname)
if obj is _MISSING:
debug('Creating', typ, 'as an independent proxy class')
obj = make_fake_object(typ, varname)
else:
debug('Creating', typ, 'with symbolic attribute assignments')
class_conditions = _CALLTREE_PARSER.get().get_class_conditions(typ)
# symbolic custom classes may assume their invariants:
if meet_class_invariants and class_conditions is not None:
for inv_condition in class_conditions.inv:
if inv_condition.evaluate is None:
continue
isok = False
with ExceptionFilter() as efilter:
isok = inv_condition.evaluate({'self': obj})
if efilter.user_exc:
raise IgnoreAttempt(
f'Class proxy could not meet invariant "{inv_condition.expr_source}" on '
f'{varname} (proxy of {typ}) because it raised: {repr(efilter.user_exc[0])}'
)
else:
if efilter.ignore or not isok:
raise IgnoreAttempt('Class proxy did not meet invariant ',
inv_condition.expr_source)
return obj
def unpack_sig(t, r, e):
args, kwargs = unpack_signature(sig, r, e)
debug(' attempting to create', fn, args, kwargs)
try:
return fn(*args, **kwargs)
except BaseException as e:
raise InputNotUnpackableError(e)
def unpacker_for_type(typ: Type) -> Callable:
unpacker = unpack_type_literals.get(typ)
if unpacker is not None:
return unpacker
# Try for a non-exact type match:
root_type = getattr(typ, '__origin__', typ)
if root_type is Union:
return (lambda t, r, e: unpack_type(
type_param(t,
r(1)[0] % len(t.__args__)), r, e))
for curtype, curhandler in unpack_type_literals.items():
if getattr(curtype, '_is_protocol', False):
continue
if curtype is Any:
continue
matches = type_matches(root_type, curtype)
if matches:
debug(' matches: ', typ, curtype, matches)
return curhandler
# attempt to create one from constructor arguments:
debug(' init', typ.__init__)
if typ.__init__ is object.__init__:
return lambda t, r, e: typ()
sig = signature(typ.__init__)
sig = inspect.Signature(
[p for (k, p) in sig.parameters.items() if k != 'self'])
return make_invocation_unpacker(typ, sig)
def summarize_execution(
fn: Callable,
args: Sequence[object] = (),
kwargs: Mapping[str, object] = None,
detach_path: bool = True,
) -> ExecutionResult:
if not kwargs:
kwargs = {}
ret = None
exc = None
try:
symbolic_ret = fn(*args, **kwargs)
if detach_path:
context_statespace().detach_path()
_ret = realize(symbolic_ret)
# TODO, this covers up potential issues with return types. Handle differently?
# summarize iterators as the values they produce:
if hasattr(_ret, "__next__"):
ret = list(_ret)
else:
ret = _ret
except BaseException as e:
if isinstance(e, (UnexploredPath, IgnoreAttempt)):
raise
if in_debug():
debug("hit exception:", type(e), e, test_stack(e.__traceback__))
exc = e
if detach_path:
context_statespace().detach_path()
args = tuple(realize(a) for a in args)
kwargs = {k: realize(v) for (k, v) in kwargs.items()}
return ExecutionResult(ret, exc, args, kwargs)
def pool_worker_main(item: WorkItemInput, output: multiprocessing.queues.Queue) -> None:
try:
# TODO figure out a more reliable way to suppress this. Redirect output?
# Ignore ctrl-c in workers to reduce noisy tracebacks (the parent will kill us):
signal.signal(signal.SIGINT, signal.SIG_IGN)
if hasattr(os, 'nice'): # analysis should run at a low priority
os.nice(10)
set_debug(False)
filename, options, deadline = item
stats: Counter[str] = Counter()
options.stats = stats
_, module_name = extract_module_from_file(filename)
try:
module = load_by_qualname(module_name)
except NotFound:
return
except ErrorDuringImport as e:
orig, frame = e.args
message = AnalysisMessage(MessageType.IMPORT_ERR, str(orig), frame.filename, frame.lineno, 0, '')
output.put((filename, stats, [message]))
debug(f'Not analyzing "{filename}" because import failed: {e}')
return
messages = analyze_any(module, options)
output.put((filename, stats, messages))
except BaseException as e:
raise CrosshairInternal(
'Worker failed while analyzing ' + filename) from e
def get_input_description(fn_name: str,
bound_args: inspect.BoundArguments,
return_val: object = _MISSING,
addl_context: str = '') -> str:
with eval_friendly_repr():
call_desc = ''
if return_val is not _MISSING:
try:
repr_str = repr(return_val)
except Exception as e:
if isinstance(e, (IgnoreAttempt, UnexploredPath)):
raise
debug(f'Exception attempting to repr function output: ', traceback.format_exc())
repr_str = _UNABLE_TO_REPR
if repr_str != 'None':
call_desc = call_desc + ' (which returns ' + repr_str + ')'
messages: List[str] = []
for argname, argval in list(bound_args.arguments.items()):
try:
repr_str = repr(argval)
except Exception as e:
if isinstance(e, (IgnoreAttempt, UnexploredPath)):
raise
debug(f'Exception attempting to repr input "{argname}": ', traceback.format_exc())
repr_str = _UNABLE_TO_REPR
messages.append(argname + ' = ' + repr_str)
call_desc = fn_name + '(' + ', '.join(messages) + ')' + call_desc
if addl_context:
return addl_context + ' when calling ' + call_desc # ' and '.join(messages)
elif messages:
return 'when calling ' + call_desc # ' and '.join(messages)
else:
return 'for any input'
def analyze_function(
fn: Callable,
options: AnalysisOptions = _DEFAULT_OPTIONS,
self_type: Optional[type] = None) -> List[AnalysisMessage]:
debug('Analyzing ', fn.__name__)
all_messages = MessageCollector()
if self_type is not None:
class_conditions = get_class_conditions(self_type)
conditions = class_conditions.methods[fn.__name__]
else:
conditions = get_fn_conditions(fn, self_type=self_type)
if conditions is None:
debug('Skipping ', str(fn),
': Unable to determine the function signature.')
return []
for syntax_message in conditions.syntax_messages():
all_messages.append(
AnalysisMessage(MessageType.SYNTAX_ERR, syntax_message.message,
syntax_message.filename, syntax_message.line_num,
0, ''))
conditions = conditions.compilable()
for post_condition in conditions.post:
messages = analyze_single_condition(
fn, options, replace(conditions, post=[post_condition]))
all_messages.extend(messages)
return all_messages.get()
def solver_is_sat(solver, *a) -> bool:
ret = solver.check(*a)
if ret == z3.unknown:
debug('Unknown satisfiability. Solver state follows:\n',
solver.sexpr())
raise UnknownSatisfiability
return ret == z3.sat
def symbolic_run(
self, fn: Callable[[TrackingStateSpace], bool]
) -> Tuple[object, Optional[BaseException]]:
search_root = SinglePathNode(True)
patched_builtins = PatchedBuiltins(contracted_builtins.__dict__,
enabled=lambda: True)
with patched_builtins:
for itr in range(1, 200):
debug('iteration', itr)
space = TrackingStateSpace(time.time() + 10.0,
1.0,
search_root=search_root)
try:
return (realize(fn(space)), None)
except IgnoreAttempt as e:
debug('ignore iteration attempt: ', str(e))
pass
except BaseException as e:
#traceback.print_exc()
return (None, e)
top_analysis, space_exhausted = space.bubble_status(
CallAnalysis())
if space_exhausted:
return (
None,
CrosshairInternal(f'exhausted after {itr} iterations'))
return (None,
CrosshairInternal(
'Unable to find a successful symbolic execution'))
def run_iteration(fn: Callable, sig: Signature,
space: StateSpace) -> Optional[PathSummary]:
with NoTracing():
args = gen_args(sig)
pre_args = copy.deepcopy(args)
ret = None
with measure_fn_coverage(fn) as coverage, ExceptionFilter() as efilter:
# coverage = lambda _: CoverageResult(set(), set(), 1.0)
# with ExceptionFilter() as efilter:
ret = fn(*args.args, **args.kwargs)
space.detach_path()
if efilter.user_exc is not None:
exc = efilter.user_exc[0]
debug("user-level exception found", repr(exc), *efilter.user_exc[1])
return PathSummary(pre_args, ret, type(exc), args, coverage(fn))
elif efilter.ignore:
return None
else:
return PathSummary(
deep_realize(pre_args),
deep_realize(ret),
None,
deep_realize(args),
coverage(fn),
)
def analyzable_filename(filename: str) -> bool:
'''
>>> analyzable_filename('foo23.py')
True
>>> analyzable_filename('#foo.py')
False
>>> analyzable_filename('23foo.py')
False
>>> analyzable_filename('setup.py')
False
'''
if not filename.endswith('.py'):
return False
lead_char = filename[0]
if (not lead_char.isalpha()) and (not lead_char.isidentifier()):
# (skip temporary editor files, backups, etc)
debug(
f'Skipping {filename} because it begins with a special character.')
return False
if filename in ('setup.py', ):
debug(
f'Skipping {filename} because files with this name are not usually import-able.'
)
return False
return True
def check_file_changed(
self,
curfile: str) -> Tuple[List[WatchedMember], List[AnalysisMessage]]:
members = []
path_to_root, module_name = extract_module_from_file(curfile)
debug(f'Attempting to import {curfile} as module "{module_name}"')
if path_to_root not in sys.path:
sys.path.append(path_to_root)
try:
module = importlib.import_module(module_name)
importlib.reload(module)
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
assert exc_traceback is not None
lineno = exception_line_in_file(
traceback.extract_tb(exc_traceback), curfile)
if lineno is None:
lineno = 1
debug(
f'Unable to load module "{module_name}" in {curfile}: {exc_type}: {exc_value}'
)
return ([], [
AnalysisMessage(MessageType.IMPORT_ERR, str(exc_value),
curfile, lineno, 0, '')
])
for (name, member) in analyzable_members(module):
qualname = module.__name__ + '.' + member.__name__
src = inspect.getsource(member)
members.append(WatchedMember(qualname, src))
return (members, [])
def analyze_single_condition(
options: AnalysisOptions,
conditions: Conditions) -> Sequence[AnalysisMessage]:
debug('Analyzing postcondition: "', conditions.post[0].expr_source, '"')
debug('assuming preconditions: ',
','.join([p.expr_source for p in conditions.pre]))
options.deadline = time.monotonic() + options.per_condition_timeout
with _CALLTREE_PARSER.open(options.condition_parser()):
analysis = analyze_calltree(options, conditions)
(condition, ) = conditions.post
addl_ctx = (' ' +
condition.addl_context if condition.addl_context else '') + '.'
if analysis.verification_status is VerificationStatus.UNKNOWN:
message = 'Not confirmed' + addl_ctx
analysis.messages = [
AnalysisMessage(MessageType.CANNOT_CONFIRM, message,
condition.filename, condition.line, 0, '')
]
elif analysis.verification_status is VerificationStatus.CONFIRMED:
message = 'Confirmed over all paths' + addl_ctx
analysis.messages = [
AnalysisMessage(MessageType.CONFIRMED, message, condition.filename,
condition.line, 0, '')
]
return analysis.messages
def analyze_module(module: types.ModuleType, options: AnalysisOptions) -> List[AnalysisMessage]:
debug('Analyzing module ', module)
messages = MessageCollector()
for (name, member) in analyzable_members(module):
messages.extend(analyze_any(member, options))
message_list = messages.get()
debug('Module', module.__name__, 'has', len(message_list), 'messages')
return message_list
def _close(self):
''' post[self]: True '''
if self.cur_file is None:
return
try:
self.cur_file.close()
except:
debug(f'WARNING: unable to close tmp file "{self.cur_file}"')
self.cur_file = None
def get_resolved_signature(fn: Callable) -> inspect.Signature:
wrapped = IdentityWrapper(fn)
if wrapped not in _RESOLVED_FNS:
_RESOLVED_FNS.add(wrapped)
try:
fn.__annotations__ = get_type_hints(fn)
except Exception as e:
debug('Could not resolve annotations on', fn, ':', e)
return inspect.signature(fn)
def get_class_conditions(cls: type) -> ClassConditions:
try:
filename = inspect.getsourcefile(cls)
except TypeError: # raises TypeError for builtins
filename = None
if filename is None:
return ClassConditions([], {})
namespace = sys.modules[cls.__module__].__dict__
super_conditions = merge_class_conditions(
[get_class_conditions(base) for base in cls.__bases__])
super_methods = super_conditions.methods
inv = super_conditions.inv[:]
parse = parse_sections(list(get_doc_lines(cls)), ('inv', ), filename)
for line_num, line in parse.sections['inv']:
inv.append(ConditionExpr(filename, line_num, line, namespace))
methods = {}
for method_name, method in cls.__dict__.items():
if not inspect.isfunction(method):
continue
conditions = get_fn_conditions(method, self_type=cls)
if conditions is None:
debug('Skipping ', str(method),
': Unable to determine the function signature.')
continue
if method_name in super_methods:
conditions = merge_fn_conditions(conditions,
super_methods[method_name])
context_string = 'when calling ' + method_name
local_inv = []
for cond in inv:
local_inv.append(
ConditionExpr(cond.filename, cond.line, cond.expr_source,
namespace, context_string))
if method_name == '__new__':
# invariants don't apply (__new__ isn't passed a concrete instance)
use_pre, use_post = False, False
elif method_name == '__del__':
use_pre, use_post = True, False
elif method_name == '__init__':
use_pre, use_post = False, True
elif method_name.startswith('__') and method_name.endswith('__'):
use_pre, use_post = True, True
elif method_name.startswith('_'):
use_pre, use_post = False, False
else:
use_pre, use_post = True, True
if use_pre:
conditions.pre.extend(local_inv)
if use_post:
conditions.post.extend(local_inv)
if conditions.has_any():
methods[method_name] = conditions
return ClassConditions(inv, methods)
def unpack_tuple(t, r, e):
args = getattr(t, '__args__', None)
if not args:
return tuple(unpack_generator(Any, r, e))
elif len(args) == 2 and args[-1] == ...:
ret = tuple(unpack_generator(args[0], r, e))
debug('tup ret', ret)
return ret
else:
return tuple(unpack_type(args[i], r, e) for i in range(len(args)))
def __init__(self, rand: random.Random, expr: z3.ExprRef,
solver: z3.Solver):
if not solver_is_sat(solver):
debug("Solver unexpectedly unsat; solver state:", solver.sexpr())
raise CrosshairInternal("Unexpected unsat from solver")
self.condition_value = solver.model().evaluate(expr,
model_completion=True)
self._stats_key = f"realize_{expr}" if z3.is_const(expr) else None
WorstResultNode.__init__(self, rand, expr == self.condition_value,
solver)
def condition_parser(
analysis_kinds: Sequence[AnalysisKind],
) -> ContextManager[ConditionParser]:
current = _CALLTREE_PARSER.get_if_in_scope()
if current is not None:
return contextlib.nullcontext(current)
debug("Using parsers: ", analysis_kinds)
condition_parser = CompositeConditionParser()
condition_parser.parsers.extend(_PARSER_MAP[k](condition_parser)
for k in analysis_kinds)
return _CALLTREE_PARSER.open(condition_parser)
def describe_behavior(
fn: Callable, args: inspect.BoundArguments) -> Tuple[object, Optional[Exception]]:
with ExceptionFilter() as efilter:
ret = fn(*args.args, **args.kwargs)
return (ret, None)
if efilter.user_exc is not None:
debug('user-level exception found', *efilter.user_exc)
return (None, efilter.user_exc[0])
if efilter.ignore:
return (None, IgnoreAttempt())
assert False
def _prune_workers(self, curtime):
for worker, item in self._workers:
(_, _, deadline) = item
if worker.is_alive() and curtime > deadline:
debug('Killing worker over deadline', worker)
worker.terminate()
time.sleep(0.5)
if worker.is_alive():
worker.kill()
worker.join()
self._workers = [(w, i) for w, i in self._workers if w.is_alive()]