def queries_equivalent(q1: Query, q2: Query, state_vars: [EVar], extern_funcs: {str: TFunc}, assumptions: Exp = T): with task("checking query equivalence", q1=q1.name, q2=q2.name): if q1.ret.type != q2.ret.type: return False q1args = dict(q1.args) q2args = dict(q2.args) if q1args != q2args: return False args = FrozenDict(q1args) key = (args, assumptions) checker = _qe_cache.get(key) if checker is None: checker = ModelCachingSolver(vars=list( itertools.chain(state_vars, (EVar(v).with_type(t) for v, t in args.items()))), funcs=extern_funcs, assumptions=assumptions) _qe_cache[key] = checker q1a = EAll(q1.assumptions) q2a = EAll(q2.assumptions) return checker.valid(EEq(q1a, q2a)) and checker.valid( EImplies(q1a, EEq(q1.ret, q2.ret)))
def queries_equivalent(q1: Query, q2: Query, state_vars: [EVar], extern_funcs: {str: TFunc}, assumptions: Exp = ETRUE): """Determine whether two queries always return the same result. This function also checks that the two queries have semantically equivalent preconditions. Checking the preconditions is necessary to ensure semantic equivalence of the queries: a query object should be interpreted to mean "if my preconditions hold then I compute and return my body expression". If two queries do not have semantically equivalent preconditions, then there might be cases where one is obligated to return a value and the other has no defined behavior. """ with task("checking query equivalence", q1=q1.name, q2=q2.name): if q1.ret.type != q2.ret.type: return False q1args = dict(q1.args) q2args = dict(q2.args) if q1args != q2args: return False checker = solver_for_context(context=RootCtx( state_vars=state_vars, args=[EVar(a).with_type(t) for (a, t) in q1.args], funcs=extern_funcs), assumptions=assumptions) q1a = EAll(q1.assumptions) q2a = EAll(q2.assumptions) return checker.valid(EEq(q1a, q2a)) and checker.valid( EImplies(q1a, EEq(q1.ret, q2.ret)))
def queries_equivalent(q1: Query, q2: Query): if q1.ret.type != q2.ret.type: return False q1args = dict(q1.args) q2args = dict(q2.args) if q1args != q2args: return False q1a = EAll(q1.assumptions) q2a = EAll(q2.assumptions) return valid(EImplies(EAny([q1a, q2a]), EEq(q1.ret, q2.ret)))
def run(self): os.makedirs(log_dir.value, exist_ok=True) with open(os.path.join(log_dir.value, "{}.log".format(self.q.name)), "w", buffering=LINE_BUFFER_MODE) as f: original_stdout = sys.stdout sys.stdout = f try: print("STARTING IMPROVEMENT JOB {}".format(self.q.name)) print(pprint(self.q)) if nice_children.value: os.nice(20) stop_callback = lambda: self.stop_requested cost_model = CostModel( funcs=self.context.funcs(), assumptions=EAll(self.assumptions), freebies=self.freebies, ops=self.ops, solver_args={"stop_callback": stop_callback}) for expr in itertools.chain( (self.q.ret, ), core.improve(target=self.q.ret, assumptions=EAll(self.assumptions), context=self.context, hints=self.hints, stop_callback=stop_callback, cost_model=cost_model, ops=self.ops, improve_count=self.improve_count)): self.solutions_q.put((self.q, expr)) print("PROVED OPTIMALITY FOR {}".format(self.q.name)) except core.StopException: print("stopping synthesis of {}".format(self.q.name)) return finally: # Restore the original stdout handle. Python multiprocessing does # some stream flushing as the process exits, and if we leave stdout # unchanged then it will refer to a closed file when that happens. sys.stdout = original_stdout
def array_in_bounds(self, elem_type, a, i): if elem_type == BOOL: len = EBinOp(EEscape("{a}.length", ["a"], [a]), "<<", ENum(6).with_type(INT)).with_type(INT) else: len = EEscape("{a}.length", ["a"], [a]).with_type(INT) return EAll([ EBinOp(i, ">=", ZERO).with_type(BOOL), EBinOp(i, "<", len).with_type(BOOL)])
def find_refinement(ast, state_map, lib, assumptions): assumptions = EAll( itertools.chain(assumptions, ast.assumptions, (EEq(EVar(v).with_type(e.type), e) for (v, e) in state_map.items()))) for (v, t) in ast.statevars: refs = list(lib.impls(EVar(v).with_type(t), assumptions=assumptions)) if not (len(refs) == 1 and refs[0] == t): return (v, refs) return None
def __init__(self, spec: Spec, concretization_functions: [(EVar, Exp)], query_specs: [Query], query_impls: OrderedDict, updates: defaultdict, handle_updates: defaultdict): """Construct an implementation. This constructor should be considered private to the `impls` module. You should call `construct_initial_implementation` instead of calling this constructor directly; this constructor makes it easy to accidentally create a malformed implementation. Parameters: - spec: the original specification - concretization_functions: pairs of (v, e) indicating that this implementation stores a state variable `v` whose value tracks `e`, a function of the state in `spec` - query_specs: specifications for all queries in `spec` plus additional private helper queries that may have been added, in terms of the state in spec - query_impls: implementations for all queries in `spec` plus additional private helper queries, in terms of the state variables in `concretization_functions`. The query implementations are stored in a map keyed by query name. - updates: a map from (concrete_var_name, op_name) to statements `stm`, where the concrete_var_name is one of the state variables described by `concretization_functions`, op_name is one of the update operations in `spec`, and `stm` is a statement that may use private helper queries and exists entirely to maintain the relationship between `concrete_var_name` and the state in the specification. The statements are all in terms of the original state, before the update started executing. - handle_updates: a map from (handle_type, op_name) to statements, where handle_type is a THandle instance and op_name is one of the update operations in `spec`. These statements update the values of every reachable instance of the given handle (aka pointer). Like updates, these statements are in terms of the original state before the update started executing. """ self.spec = spec self._concretization_functions = concretization_functions self.query_specs = query_specs self.query_impls = query_impls self.updates = updates self.handle_updates = handle_updates self.state_solver = ModelCachingSolver(vars=self.abstract_state, funcs=self.extern_funcs, assumptions=EAll( spec.assumptions))
def can_elim_vars(spec: Exp, assumptions: Exp, vs: [EVar]): """Does any execution of `spec` actually depend on any of `vs`? It is possible for a variable to appear in an expression like `spec` without affecting its value. This function uses the solver to determine whether any of the given variables can affect the output of `spec`. """ spec = strip_EStateVar(spec) sub = {v.id: fresh_var(v.type) for v in vs} return valid( EImplies(EAll([assumptions, subst(assumptions, sub)]), EEq(spec, subst(spec, sub))))
def run(self): print("STARTING IMPROVEMENT JOB {}".format(self.q.name)) os.makedirs(log_dir.value, exist_ok=True) with open(os.path.join(log_dir.value, "{}.log".format(self.q.name)), "w", buffering=LINE_BUFFER_MODE) as f: sys.stdout = f print("STARTING IMPROVEMENT JOB {}".format(self.q.name)) print(pprint(self.q)) if nice_children.value: os.nice(20) cost_model = CostModel(funcs=self.context.funcs(), assumptions=EAll(self.assumptions), freebies=self.freebies, ops=self.ops) try: for expr in itertools.chain( (self.q.ret, ), core.improve(target=self.q.ret, assumptions=EAll(self.assumptions), context=self.context, hints=self.hints, stop_callback=lambda: self.stop_requested, cost_model=cost_model, ops=self.ops, improve_count=self.improve_count)): new_rep, new_ret = unpack_representation(expr) self.k(new_rep, new_ret) print("PROVED OPTIMALITY FOR {}".format(self.q.name)) except core.StopException: print("stopping synthesis of {}".format(self.q.name)) return
def array_resize_for_index(self, elem_type, a, i): """Resize the array until `i` is a legal index. When i < 0, it will do nothing instead. """ new_a = fresh_name(hint="new_array") if elem_type == BOOL: t = "long" else: t = self.strip_generics(self.visit(elem_type, name="")) len = EEscape("{a}.length", ["a"], [a]).with_type(INT) double_and_incr_size = SEscape( "{{indent}}{t}[] {new_a} = new {t}[({{len}} << 1) + 1];\n{{indent}}System.arraycopy({{a}}, 0, {new_a}, 0, {{len}});\n{{indent}}{{a}} = {new_a};\n".format(t=t, new_a=new_a), ["a", "len"], [a, len]) self.visit(SWhile( EAll([EBinOp(i, ">=", ZERO).with_type(BOOL), ENot(self.array_in_bounds(elem_type, a, i))]), double_and_incr_size))
def exp_wf_nonrecursive(solver, e: Exp, context: Context, pool=RUNTIME_POOL, assumptions: Exp = ETRUE): """Check the well-formedness of `e` but do not recurse into its children. Returns True or an instance of No explaining why `e` is not well-formed. See `exp_wf` for an explanation of well-formedness and the parameters that this function requires. """ if hasattr(e, "_wf"): return True state_vars = OrderedSet(v for v, p in context.vars() if p == STATE_POOL) args = OrderedSet(v for v, p in context.vars() if p == RUNTIME_POOL) h = extension_handler(type(e)) if h is not None: assumptions = EAll([assumptions, context.path_condition()]) msg = h.check_wf(e, state_vars=state_vars, args=args, pool=pool, assumptions=assumptions, is_valid=solver.valid) if msg is not None: return No(msg) e._wf = True return True at_runtime = pool == RUNTIME_POOL if isinstance(e, EStateVar) and not at_runtime: return No("EStateVar in state pool position") if isinstance(e, EVar): if at_runtime and e in state_vars: return No("state var at runtime") elif not at_runtime and e in args: return No("arg in state exp") e._wf = True return True
def define_type(self, toplevel_name, t, name, sharing): if isinstance(t, TEnum): self.begin_statement() self.write("enum ", name, " ") with self.block(): for case in t.cases: self.begin_statement() self.write(case, ",") self.end_statement() self.write(";") self.end_statement() elif isinstance(t, THandle): fields = [("val", t.value_type)] self.begin_statement() self.write("struct ", name, " ") with self.block(): with self.deindented(): self.write_stmt("public:") for (f, ft) in fields: self.declare_field(f, ft) with self.deindented(): self.write_stmt("private:") self.write(";") self.end_statement() # s = "struct {name} {{\n".format(indent=indent, name=name) # s += "public:\n".format(indent=indent) # for (f, ft) in fields: # s += self.declare_field(f, ft) # s += "private:\n".format(indent=indent) # s += "friend class {toplevel_name};\n".format(indent=indent+INDENT, toplevel_name=toplevel_name) # for group in sharing.get(t, []): # s += "union {{\n".format(indent=indent+INDENT) # for gt in group: # intrusive_data = gt.intrusive_data(t) # s += "struct {{\n".format(indent=indent+INDENT*2) # for (f, ft) in intrusive_data: # s += "{field_decl};\n".format(indent=indent+INDENT*3, field_decl=self.visit(ft, f)) # s += "}};\n".format(indent=indent+INDENT*2) # s += "}};\n".format(indent=indent+INDENT) # s += "}};\n".format(indent=indent) # return s elif isinstance(t, TRecord): self.begin_statement() self.write("struct ", name, " ") with self.block(): # TODO: sort fields by size descending for better packing for f, ft in t.fields: self.declare_field(f, ft) self.write_stmt("inline ", name, "() { }") self.begin_statement() self.write("inline ", name, "(") self.visit_args([("_" + f, t) for (f, t) in t.fields]) self.write(") : ") for i, (f, ft) in enumerate(t.fields): if i > 0: self.write(", ") self.write(f, "(::std::move(_", f, "))") self.write(" { }") self.end_statement() self.begin_statement() self.write("inline bool operator==(const ", name, "& other) const ") with self.block(): this = EEscape("(*this)", (), ()).with_type(t) other = EVar("other").with_type(t) r = self.visit( EAll([ EEq( EGetField(this, f).with_type(ft), EGetField(other, f).with_type(ft)) for f, ft in t.fields ])) self.begin_statement() self.write("return ", r, ";") self.end_statement() self.end_statement() self.write(";") self.end_statement() elif isinstance(t, TTuple): return self.define_type( toplevel_name, TRecord( tuple( ("_{}".format(i), t.ts[i]) for i in range(len(t.ts)))), name, sharing) else: return ""
def define_type(self, toplevel_name, t, name, sharing): if isinstance(t, TEnum): self.begin_statement() self.write("public enum ", name, " ") with self.block(): for case in t.cases: self.begin_statement() self.write(case) self.end_statement() self.end_statement() elif isinstance(t, THandle) or isinstance(t, TRecord): public_fields = [] private_fields = [] value_equality = True handle_val_is_this = False if isinstance(t, THandle): if isinstance(t.value_type, TRecord): handle_val_is_this = True else: public_fields = [("val", t.value_type)] value_equality = False for group in sharing.get(t, []): for gt in group: intrusive_data = gt.intrusive_data(t) for (f, ft) in intrusive_data: private_fields.append((f, ft)) else: public_fields = list(t.fields) all_fields = public_fields + private_fields self.begin_statement() self.write("public static class ", name) if handle_val_is_this: self.write(" extends ", self.visit(t.value_type, "")) self.write(" implements java.io.Serializable ") with self.block(): for (f, ft) in public_fields + private_fields: self.begin_statement() self.write("private {field_decl};".format(field_decl=self.visit(ft, f))) self.end_statement() for (f, ft) in public_fields: self.begin_statement() self.write("public {type} get{Field}() {{ return {field}; }}".format( type=self.visit(ft, ""), Field=common.capitalize(f), field=f)) self.end_statement() if handle_val_is_this: self.begin_statement() self.write("public {type} getVal() {{ return this; }}".format( type=self.visit(t.value_type, ""))) self.end_statement() def flatten(field_types): args = [] exps = [] for ft in field_types: if isinstance(ft, TRecord): aa, ee = flatten([t for (f, t) in ft.fields]) args.extend(aa) exps.append(EMakeRecord(tuple((f, e) for ((f, _), e) in zip(ft.fields, ee))).with_type(ft)) elif isinstance(ft, TTuple): aa, ee = flatten(ft.ts) args.extend(aa) exps.append(ETuple(tuple(ee)).with_type(ft)) else: v = self.fv(ft, "v") args.append((v.id, ft)) exps.append(v) return args, exps if isinstance(t, THandle): args, exps = flatten([ft for (f, ft) in (t.value_type.fields if handle_val_is_this else public_fields)]) else: args = public_fields exps = [EVar(f) for (f, ft) in args] self.begin_statement() self.write("public {ctor}({args}) ".format(ctor=name, args=", ".join(self.visit(ft, f) for (f, ft) in args))) with self.block(): if handle_val_is_this: es = [self.visit(e) for e in exps] self.begin_statement() self.write("super({args});\n".format( args=", ".join(es))) for ((f, ft), e) in zip(public_fields, exps): e = self.visit(e) self.begin_statement() self.write("this.{f} = {e};\n".format(f=f, e=e)) self.end_statement() if value_equality: self.begin_statement() self.write("@Override") self.end_statement() self.begin_statement() self.write("public int hashCode() ") with self.block(): (compute, hc) = self.compute_hash(public_fields + private_fields) self.write(compute) self.begin_statement() self.write("return ", hc, ";") self.end_statement() self.end_statement() self.begin_statement() self.write("@Override") self.end_statement() self.begin_statement() self.write("public boolean equals(Object other) ") with self.block(): self.write(self.get_indent(), "if (other == null) return false;\n") self.write(self.get_indent(), "if (other == this) return true;\n") self.write(self.get_indent(), "if (!(other instanceof {name})) return false;\n".format(name=name)) self.write(self.get_indent(), "{name} o = ({name})other;\n".format(name=name)) eq = self.visit(EAll([EEq( EEscape("this.{}".format(f), (), ()).with_type(ft), EEscape("o.{}".format(f), (), ()).with_type(ft)) for (f, ft) in all_fields])) self.write(self.get_indent(), "return ", eq, ";\n") self.end_statement() self.end_statement() elif isinstance(t, TTuple): return self.define_type(toplevel_name, TRecord(tuple(("_{}".format(i), t.ts[i]) for i in range(len(t.ts)))), name, sharing) else: return ""
def path_condition(self): return EAll(self.path_conditions())
def can_elim_vars(spec: Exp, assumptions: Exp, vs: [EVar]): spec = strip_EStateVar(spec) sub = {v.id: fresh_var(v.type) for v in vs} return valid( EImplies(EAll([assumptions, subst(assumptions, sub)]), EEq(spec, subst(spec, sub))))
def _add_subquery(self, sub_q: Query, used_by: Stm) -> Stm: """Add a query that helps maintain some other state. Parameters: sub_q - the specification of the helper query used_by - the statement that calls `sub_q` If a query already exists that is equivalent to `sub_q`, this method returns `used_by` rewritten to use the existing query and does not add the query to the implementation. Otherwise it returns `used_by` unchanged. """ with task("adding query", query=sub_q.name): sub_q = shallow_copy(sub_q) with task("checking whether we need more handle assumptions"): new_a = implicit_handle_assumptions( reachable_handles_at_method(self.spec, sub_q)) if not valid(EImplies(EAll(sub_q.assumptions), EAll(new_a))): event("we do!") sub_q.assumptions = list( itertools.chain(sub_q.assumptions, new_a)) with task("repairing state var boundaries"): extra_available_state = [ e for v, e in self._concretization_functions ] sub_q.ret = repair_well_formedness( strip_EStateVar(sub_q.ret), self.context_for_method(sub_q), extra_available_state) with task("simplifying"): orig_a = sub_q.assumptions orig_a_size = sum(a.size() for a in sub_q.assumptions) orig_ret_size = sub_q.ret.size() sub_q.assumptions = tuple( simplify_or_ignore(a) for a in sub_q.assumptions) sub_q.ret = simplify(sub_q.ret) a_size = sum(a.size() for a in sub_q.assumptions) ret_size = sub_q.ret.size() event("|assumptions|: {} -> {}".format(orig_a_size, a_size)) event("|ret|: {} -> {}".format(orig_ret_size, ret_size)) if a_size > orig_a_size: print("NO, BAD SIMPLIFICATION") print("original") for a in orig_a: print(" - {}".format(pprint(a))) print("simplified") for a in sub_q.assumptions: print(" - {}".format(pprint(a))) assert False state_vars = self.abstract_state funcs = self.extern_funcs qq = find_one( self.query_specs, lambda qq: dedup_queries.value and queries_equivalent(qq, sub_q, state_vars=state_vars, extern_funcs=funcs, assumptions=EAll(self.abstract_invariants))) if qq is not None: event("subgoal {} is equivalent to {}".format( sub_q.name, qq.name)) arg_reorder = [[x[0] for x in sub_q.args].index(a) for (a, t) in qq.args] class Repl(BottomUpRewriter): def visit_ECall(self, e): args = tuple(self.visit(a) for a in e.args) if e.func == sub_q.name: args = tuple(args[idx] for idx in arg_reorder) return ECall(qq.name, args).with_type(e.type) else: return ECall(e.func, args).with_type(e.type) used_by = Repl().visit(used_by) else: self.add_query(sub_q) return used_by
def path_condition(self) -> Exp: """Return an expression capturing all of self.path_conditions()""" return EAll(self.path_conditions())
def possibly_useful_nonrecursive( solver, e: Exp, context: Context, pool=RUNTIME_POOL, assumptions: Exp = ETRUE, ops: [Op] = ()) -> bool: """Heuristic filter to ignore expressions that are almost certainly useless.""" state_vars = OrderedSet(v for v, p in context.vars() if p == STATE_POOL) args = OrderedSet(v for v, p in context.vars() if p == RUNTIME_POOL) assumptions = EAll([assumptions, context.path_condition()]) at_runtime = pool == RUNTIME_POOL h = extension_handler(type(e)) if h is not None: res = h.possibly_useful(e, context, pool, assumptions, ops, solver) if not res: return res if isinstance(e, EStateVar) and not free_vars(e.e): return No("constant value in state position") if (isinstance(e, EDropFront) or isinstance(e, EDropBack)) and not at_runtime: return No("EDrop* in state position") if not allow_big_sets.value and isinstance(e, EFlatMap) and not at_runtime: return No("EFlatMap in state position") if not allow_int_arithmetic_state.value and not at_runtime and isinstance( e, EBinOp) and e.type == INT: return No("integer arithmetic in state position") if is_collection(e.type) and not is_scalar(e.type.elem_type): return No("collection of nonscalar: e {}\n elem_type: {}\n".format( e, e.type.elem_type)) if isinstance(e.type, TMap) and not is_scalar(e.type.k): return No("bad key type {}".format(pprint(e.type.k))) if isinstance(e.type, TMap) and isinstance(e.type.v, TMap): return No("map to map") # This check is probably a bad idea: whether `the` is legal may depend on # the contex that the expression is embedded within, so we can't skip it # during synthesis just because it looks invalid now. # if isinstance(e, EUnaryOp) and e.op == UOp.The: # len = EUnaryOp(UOp.Length, e.e).with_type(INT) # if not valid(EImplies(assumptions, EBinOp(len, "<=", ENum(1).with_type(INT)).with_type(BOOL))): # return No("illegal application of 'the': could have >1 elems") if not at_runtime and isinstance( e, EBinOp) and e.op == "-" and is_collection(e.type): return No("collection subtraction in state position") # if not at_runtime and isinstance(e, ESingleton): # return No("singleton in state position") if not allow_nonzero_state_constants.value and not at_runtime and isinstance( e, ENum) and e.val != 0: return No("nonzero integer constant in state position") if not allow_binop_state.value and at_runtime and isinstance( e, EStateVar) and isinstance(e.e, EBinOp) and is_scalar( e.e.e1.type) and is_scalar(e.e.e2.type): return No( "constant-time binary operator {!r} in state position".format( e.e.op)) if not allow_conditional_state.value and not at_runtime and isinstance( e, ECond): return No("conditional in state position") if isinstance(e, EMakeMap2) and isinstance(e.e, EEmptyList): return No("trivially empty map") if isinstance(e, EMakeMap2) and isinstance(e.e, ESingleton): return No("really tiny map") if not at_runtime and (isinstance(e, EArgMin) or isinstance(e, EArgMax)): # Cozy has no way to efficiently implement mins/maxes when more than # one element may leave the collection. from cozy.state_maintenance import mutate for op in ops: elems = e.e elems_prime = mutate(elems, op.body) formula = EAll([assumptions] + list(op.assumptions) + [ EGt( ELen( EBinOp(elems, "-", elems_prime).with_type(elems.type)), ONE) ]) if solver.satisfiable(formula): return No( "more than one element might be removed during {}".format( op.name)) if not allow_peels.value and not at_runtime and isinstance(e, EFilter): # catch "peels": removal of zero or one elements if solver.valid( EImplies( assumptions, ELe( ELen( EFilter( e.e, ELambda(e.predicate.arg, ENot( e.predicate.body))).with_type(e.type)), ONE))): return No("filter is a peel") if not allow_big_maps.value and not at_runtime and isinstance( e, EMakeMap2) and is_collection(e.type.v): all_collections = [sv for sv in state_vars if is_collection(sv.type)] total_size = ENum(0).with_type(INT) for c in all_collections: total_size = EBinOp(total_size, "+", EUnaryOp(UOp.Length, c).with_type(INT)).with_type(INT) my_size = EUnaryOp( UOp.Length, EFlatMap( EUnaryOp(UOp.Distinct, e.e).with_type(e.e.type), e.value_function).with_type(e.type.v)).with_type(INT) s = EImplies(assumptions, EBinOp(total_size, ">=", my_size).with_type(BOOL)) if not solver.valid(s): return No("non-polynomial-sized map") return True