def watch(self, new_target): print("watching new target...") self.backlog_counter = 0 self.target = new_target self.roots = OrderedSet() types = OrderedSet() for e in itertools.chain(all_exps(new_target), *[all_exps(h) for h in self.hints]): if isinstance(e, ELambda): continue for pool in ALL_POOLS: exp = e if pool == STATE_POOL: exp = strip_EStateVar(e) fvs = free_vars(exp) if all(v in self.legal_free_vars for v in fvs) and self.is_legal_in_pool(exp, pool): _on_exp(exp, "new root", pool_name(pool)) exp._root = True self.roots.add((exp, pool)) if pool == STATE_POOL and all(v in self.state_vars for v in fvs): self.roots.add((EStateVar(exp).with_type(exp.type), RUNTIME_POOL)) types.add(exp.type) else: _on_exp(exp, "rejected root", pool_name(pool)) for b in self.binders: types.add(b.type) for t in types: self.roots.add((construct_value(t), RUNTIME_POOL)) self.roots = list(self.roots) self.roots.sort(key = lambda tup: tup[0].size()) self._watches = group_by( enumerate_fragments2(new_target), k=lambda ctx: (ctx.pool, ctx.e.type), v=lambda ctxs: sorted(ctxs, key=lambda ctx: -ctx.e.size())) print("done!")
def exp_wf_nonrecursive(solver, e: Exp, context: Context, pool=RUNTIME_POOL, assumptions: Exp = T): state_vars = OrderedSet(v for v, p in context.vars() if p == STATE_POOL) args = OrderedSet(v for v, p in context.vars() if p == RUNTIME_POOL) assumptions = EAll([assumptions, context.path_condition()]) h = extension_handler(type(e)) if h is not None: msg = h.check_wf(e, state_vars=state_vars, args=args, pool=pool, assumptions=assumptions, is_valid=solver.valid) if msg is not None: raise ExpIsNotWf(e, e, msg) return at_runtime = pool == RUNTIME_POOL if isinstance(e, EStateVar) and not at_runtime: raise ExpIsNotWf(e, e, "EStateVar in state pool position") if isinstance(e, EVar): if at_runtime and e in state_vars: raise ExpIsNotWf(e, e, "state var at runtime") elif not at_runtime and e in args: raise ExpIsNotWf(e, e, "arg in state exp")
def cleanup(self): """ Remove unused state, queries, and updates. """ # sort of like mark-and-sweep queries_to_keep = OrderedSet(q.name for q in self.query_specs if q.visibility == Visibility.Public) state_vars_to_keep = OrderedSet() changed = True while changed: changed = False for qname in list(queries_to_keep): if qname in self.query_impls: for sv in free_vars(self.query_impls[qname]): if sv not in state_vars_to_keep: state_vars_to_keep.add(sv) changed = True for e in all_exps(self.query_impls[qname].ret): if isinstance(e, ECall): if e.func not in queries_to_keep: queries_to_keep.add(e.func) changed = True for op in self.op_specs: for ((ht, op_name), code) in self.handle_updates.items(): if op.name == op_name: for qname in _queries_used_by(code): if qname not in queries_to_keep: queries_to_keep.add(qname) changed = True for sv in state_vars_to_keep: for qname in _queries_used_by(self.updates[(sv, op.name)]): if qname not in queries_to_keep: queries_to_keep.add(qname) changed = True # remove old specs for q in list(self.query_specs): if q.name not in queries_to_keep: self.query_specs.remove(q) # remove old implementations for qname in list(self.query_impls.keys()): if qname not in queries_to_keep: del self.query_impls[qname] # remove old state vars self.concrete_state = [ v for v in self.concrete_state if any(v[0] in free_vars(q) for q in self.query_impls.values()) ] # remove old method implementations for k in list(self.updates.keys()): v, op_name = k if v not in [var for (var, exp) in self.concrete_state]: del self.updates[k]
def __init__(self, examples, cost_model: CostModel, check_wf=None, hints=None, heuristics=None, stop_callback=None, do_eviction=True): """Set up a fresh enumerator. Parameters: - examples: a set of example inputs to deduplicate expressions - cost_model: a cost model to tell us which expressions to prefer - check_wf: an optional additional filter to restrict which expressions are visited - hints: expressions that get treated as size 0 during enumeration, so they are enumerated very early - heuristics: an optional function to improve visited expressions - stop_callback: a function that is checked periodically to stop enumeration - do_eviction: boolean. if true, this class spends time trying to evict older, slower versions of expressions from its cache """ self.examples = list(examples) self.cost_model = cost_model self.cache = ExpCache() # Set of (pool, size, context) tuples that are currently being # enumerated. This is used to catch infinite recursion bugs, since # enumerating expressions in one context may require enumerating # expressions in a different context recursively. self.in_progress = set() # Set of (pool, size, context) tuples that have been fully enumerated; # there are no more expressions to discover and the results have been # cached in `cache`. There is no overlap between this and the # `in_progress` set. self.complete = set() if check_wf is None: check_wf = lambda e, ctx, pool: True self.check_wf = check_wf if hints is None: hints = () self.hints = OrderedSet( (e, ctx.generalize(free_vars(e)), p) for (e, ctx, p) in hints) self.hint_types = OrderedSet() for h, _, _ in self.hints: self.hint_types |= all_types(h) if heuristics is None: heuristics = lambda e, ctx, pool: () self.heuristics = heuristics if stop_callback is None: stop_callback = lambda: False self.stop_callback = stop_callback self.do_eviction = do_eviction self.stat_timer = Periodically(self.print_stats, timespan=datetime.timedelta(seconds=2))
def test_heap_wf(self): e = EHeapPeek2(EStateVar(EMakeMinHeap(EVar('xs'), ELambda(EVar('_var21501'), EVar('_var21501'))))) assert retypecheck(e, env={ "xs": INT_BAG, "_var21501": INT}) state_vars = OrderedSet([EVar('xs').with_type(TBag(TInt()))]) args = OrderedSet([EVar('x').with_type(TInt())]) pool = RUNTIME_POOL assert exp_wf(e, context=RootCtx(args=args, state_vars=state_vars), pool=pool)
def __init__(self, state_vars : [Exp], args : [Exp], funcs : {str:TFunc} = None): """Construct a root context. Parameters: state_vars - state variables in scope args - argument variables in scope funcs - functions in scope The sets of state variables and arguments must be disjoint. """ self.state_vars = OrderedSet(state_vars) self.args = OrderedSet(args) self.functions = OrderedDict(funcs or ()) assert not (self.state_vars & self.args)
def __init__(self, target, assumptions, binders, state_vars, args, legal_free_vars, examples, cost_model, builder, stop_callback, hints, solver): self.binders = OrderedSet(binders) self.state_vars = OrderedSet(state_vars) self.args = OrderedSet(args) self.legal_free_vars = legal_free_vars self.stop_callback = stop_callback self.cost_model = cost_model self.builder = builder self.seen = SeenSet() self.assumptions = assumptions self.hints = list(hints) self.solver = solver self.reset(examples) self.watch(target)
def test_repair_regression01(self): e = EBinOp(EMapKeys(EMakeMap2(EVar('conns').with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), ELambda(EVar('_var17561').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), EBool(True).with_type(TBool()))).with_type(TMap(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))), TBool()))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), '-', EMapKeys(ELet(EUnaryOp('the', EMap(EFilter(EVar('conns').with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), ELambda(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), EBinOp(EGetField(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), 'conn_iface').with_type(TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), '==', EVar('i').with_type(TNative('mongo::executor::ConnectionPool::ConnectionInterface*'))).with_type(TBool()))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), ELambda(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))))).with_type(TList(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))))).with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), ELambda(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), EMakeMap2(EBinOp(EBinOp(EVar('conns').with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), '-', ESingleton(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), '+', ESingleton(EMakeRecord((('conn_state', EEnumEntry('READY').with_type(TEnum(('READY', 'PROCESSING', 'CHECKED_OUT')))), ('conn_host', EGetField(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), 'conn_host').with_type(TNative('mongo::HostAndPort'))), ('conn_iface', EGetField(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), 'conn_iface').with_type(TNative('mongo::executor::ConnectionPool::ConnectionInterface*'))), ('conn_next_refresh', ECall('after', (EVar('lastUsed').with_type(TNative('mongo::Date_t')), EVar('refreshRequirement').with_type(TNative('mongo::Milliseconds')))).with_type(TNative('mongo::Date_t'))), ('conn_returned', EVar('now').with_type(TNative('mongo::Date_t'))), ('conn_last_used', EVar('retId').with_type(TInt())), ('conn_dropped', EGetField(EVar('c').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), 'conn_dropped').with_type(TBool())))).with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), ELambda(EVar('_var17561').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), EBool(True).with_type(TBool()))).with_type(TMap(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))), TBool())))).with_type(TMap(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))), TBool()))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))))).with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))) context = RootCtx(state_vars=OrderedSet([EVar('minConnections').with_type(TInt()), EVar('maxConnections').with_type(TInt()), EVar('maxConnecting').with_type(TInt()), EVar('refreshTimeout').with_type(TNative('mongo::Milliseconds')), EVar('refreshRequirement').with_type(TNative('mongo::Milliseconds')), EVar('hostTimeout').with_type(TNative('mongo::Milliseconds')), EVar('conns').with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), EVar('reqs').with_type(TBag(TRecord((('rq_callback', TNative('mongo::executor::ConnectionPool::GetConnectionCallback*')), ('rq_expiration', TNative('mongo::Date_t')), ('rq_host', TNative('mongo::HostAndPort')))))), EVar('_idleHosts').with_type(TBag(TRecord((('host_id', TNative('mongo::HostAndPort')), ('host_timeout', TNative('mongo::Date_t')))))), EVar('retId').with_type(TInt())]), args=OrderedSet([EVar('i').with_type(TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), EVar('lastUsed').with_type(TNative('mongo::Date_t')), EVar('now').with_type(TNative('mongo::Date_t'))]), funcs=OrderedDict([('eternity', TFunc((), TNative('mongo::Date_t'))), ('after', TFunc((TNative('mongo::Date_t'), TNative('mongo::Milliseconds')), TNative('mongo::Date_t'))), ('nullConn', TFunc((), TNative('mongo::executor::ConnectionPool::ConnectionInterface*'))), ('nullReq', TFunc((), TNative('mongo::executor::ConnectionPool::GetConnectionCallback*')))])) assert not exp_wf(e, context=context, pool=RUNTIME_POOL) extra_state = [EVar('reqs').with_type(TBag(TRecord((('rq_callback', TNative('mongo::executor::ConnectionPool::GetConnectionCallback*')), ('rq_expiration', TNative('mongo::Date_t')), ('rq_host', TNative('mongo::HostAndPort')))))), EVar('conns').with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), EVar('hostTimeout').with_type(TNative('mongo::Milliseconds')), EVar('refreshRequirement').with_type(TNative('mongo::Milliseconds')), EVar('retId').with_type(TInt()), EMakeMap2(EVar('conns').with_type(TBag(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))))), ELambda(EVar('_var17561').with_type(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool())))), EBool(True).with_type(TBool()))).with_type(TMap(TRecord((('conn_state', TEnum(('READY', 'PROCESSING', 'CHECKED_OUT'))), ('conn_host', TNative('mongo::HostAndPort')), ('conn_iface', TNative('mongo::executor::ConnectionPool::ConnectionInterface*')), ('conn_next_refresh', TNative('mongo::Date_t')), ('conn_returned', TNative('mongo::Date_t')), ('conn_last_used', TInt()), ('conn_dropped', TBool()))), TBool()))] e_prime = repair_well_formedness(e, context, extra_state) assert exp_wf(e_prime, context=context, pool=RUNTIME_POOL)
def eval_bulk(e, envs, bind_callback=None, use_default_values_for_undefined_vars: bool = False): e = purify(e) if bind_callback is None: bind_callback = lambda arg, val: None # return [eval(e, env, bind_callback=bind_callback) for env in envs] if not envs: return [] ops = [] vars = OrderedSet(free_vars_and_funcs(e)) types = {v.id: v.type for v in free_vars(e)} vmap = {v: i for (i, v) in enumerate(vars)} try: envs = [[(env.get(v, mkval(types[v])) if (use_default_values_for_undefined_vars and v in types) else env[v]) for v in vars] for env in envs] except KeyError: import sys print("OH NO", file=sys.stderr) print("e = {}".format(pprint(e)), file=sys.stderr) print( "eval_bulk({!r}, {!r}, use_default_values_for_undefined_vars={!r})" .format(e, envs, use_default_values_for_undefined_vars), file=sys.stderr) # import pdb # pdb.set_trace() raise _compile(e, vmap, ops, bind_callback) return [_eval_compiled(ops, env) for env in envs]
def state_used_during_update(v1: EVar) -> [EVar]: v1_update_code = self.updates[(v1, operator.name)] v1_queries = list(self.queries_used_by(v1_update_code)) res = OrderedSet() for q in v1_queries: res |= state_read_by_query[q] return res
def __init__(self, ctx: SynthCtx, state: [EVar], assumptions: [Exp], q: Query, k, hints: [Exp] = [], freebies: [Exp] = [], ops: [Op] = [], funcs: {str: TFunc} = {}): assert all( v in state for v in free_vars(q) ), "Oops, query looks malformed due to {}:\n{}\nfree_vars({})".format( [v for v in free_vars(q) if v not in state], pprint(q), repr(q)) super().__init__() self.ctx = ctx self.state = state self.assumptions = assumptions q = shallow_copy(q) q.ret = wrap_naked_statevars(q.ret, OrderedSet(state)) self.q = q self.hints = hints self.freebies = freebies self.ops = ops self.k = k self.funcs = OrderedDict(funcs)
def __init__(self, examples, cost_model: CostModel, check_wf=None, hints=None, heuristics=None, stop_callback=None, do_eviction=True): self.examples = list(examples) self.cost_model = cost_model self.cache = {} # keys -> [exp] self.seen = {} # (ctx, pool, fp) -> frontier, i.e. [exp] self.in_progress = set() if check_wf is None: check_wf = lambda e, ctx, pool: True self.check_wf = check_wf if hints is None: hints = () self.hints = OrderedSet( (e, ctx.generalize(free_vars(e)), p) for (e, ctx, p) in hints) if heuristics is None: heuristics = lambda e, ctx, pool: () self.heuristics = heuristics if stop_callback is None: stop_callback = lambda: False self.stop_callback = stop_callback self.do_eviction = do_eviction
def exp_wf_nonrecursive(solver, e: Exp, context: Context, pool=RUNTIME_POOL, assumptions: Exp = ETRUE): """Check the well-formedness of `e` but do not recurse into its children. Returns True or an instance of No explaining why `e` is not well-formed. See `exp_wf` for an explanation of well-formedness and the parameters that this function requires. """ if hasattr(e, "_wf"): return True state_vars = OrderedSet(v for v, p in context.vars() if p == STATE_POOL) args = OrderedSet(v for v, p in context.vars() if p == RUNTIME_POOL) h = extension_handler(type(e)) if h is not None: assumptions = EAll([assumptions, context.path_condition()]) msg = h.check_wf(e, state_vars=state_vars, args=args, pool=pool, assumptions=assumptions, is_valid=solver.valid) if msg is not None: return No(msg) e._wf = True return True at_runtime = pool == RUNTIME_POOL if isinstance(e, EStateVar) and not at_runtime: return No("EStateVar in state pool position") if isinstance(e, EVar): if at_runtime and e in state_vars: return No("state var at runtime") elif not at_runtime and e in args: return No("arg in state exp") e._wf = True return True
def eval_bulk(e: Exp, envs: [{ str: object }], use_default_values_for_undefined_vars: bool = False): """Evaluate an expression in many different environments. This function accepts the same arguments as `eval`, but takes a list of environments instead of just one. The call eval_bulk(e, envs) is equivalent to [eval(e, env) for env in envs]. However, using `eval_bulk` is much faster than repeatedly calling `eval` on the same expression. """ e = purify(e) if not envs: return [] ops = [] vars = OrderedSet(free_vars_and_funcs(e)) types = {v.id: v.type for v in free_vars(e)} vmap = {v: i for (i, v) in enumerate(vars)} try: envs = [[(env.get(v, mkval(types[v])) if (use_default_values_for_undefined_vars and v in types) else env[v]) for v in vars] for env in envs] except KeyError: import sys print("OH NO", file=sys.stderr) print("e = {}".format(pprint(e)), file=sys.stderr) print( "eval_bulk({!r}, {!r}, use_default_values_for_undefined_vars={!r})" .format(e, envs, use_default_values_for_undefined_vars), file=sys.stderr) raise _compile(e, vmap, ops) return [_eval_compiled(ops, env) for env in envs]
def _possible_replacements(self, e, pool, cost): """ Yields watched expressions that appear as worse versions of the given expression. There may be more than one. """ # return free_binders = OrderedSet(v for v in free_vars(e) if v in self.binders) for ctx in self._watched_contexts(pool, e.type): watched_e = ctx.e p = ctx.pool r = ctx.replace_e_with assert e.type == watched_e.type assert p == pool _on_exp(e, "considering replacement of", watched_e) # if e.type != watched_e.type: # # _on_exp(e, "wrong type") # continue # if p != pool: # # _on_exp(e, "wrong pool") # continue if e == watched_e: # _on_exp(e, "no change") continue unbound_binders = [b for b in free_binders if b not in ctx.bound_vars] if unbound_binders: _on_exp(e, "skipped exp with free binders", ", ".join(b.id for b in unbound_binders)) continue if CHECK_SUBST_COST: watched_cost = self.cost_model.cost(watched_e, pool=pool) ordering = self.compare_costs(cost, watched_cost) if ordering == Cost.WORSE: _on_exp(e, "skipped worse replacement", pool_name(pool), watched_e) continue if ordering == Cost.UNORDERED: _on_exp(e, "skipped equivalent replacement", pool_name(pool), watched_e) # print(" e1 = {!r}".format(e)) # print(" e2 = {!r}".format(watched_e)) continue # assert all(eval_bulk(self.assumptions, self.all_examples)) if all(eval_bulk(EEq(self.target, r(e)), self.all_examples)): yield (watched_e, e, ctx.facts, r) else: _on_exp(e, "visited pointless replacement", watched_e)
def __init__(self, ctx: SynthCtx, state: [EVar], assumptions: [Exp], q: Query, k, hints: [Exp] = [], examples: [dict] = None): super().__init__() self.ctx = ctx self.state = state self.assumptions = assumptions self.q = shallow_copy(q) assert all( v in state for v in free_vars(q) ), "Oops, query looks malformed due to {}:\n{}\nfree_vars({})".format( [v for v in free_vars(q) if v not in state], pprint(q), repr(q)) q.ret = wrap_naked_statevars(q.ret, OrderedSet(state)) self.hints = hints self.examples = examples self.k = k
def __init__(self, state: [EVar], assumptions: [Exp], q: Query, context: Context, solutions_q, hints: [Exp] = [], freebies: [Exp] = [], ops: [Op] = [], improve_count=None): super().__init__() self.state = state self.assumptions = assumptions q = shallow_copy(q) q.ret = wrap_naked_statevars(q.ret, OrderedSet(state)) self.q = q self.context = context self.hints = hints self.freebies = freebies self.ops = ops self.solutions_q = solutions_q self.improve_count = improve_count
def max_storage_size(e, freebies: [Exp] = []): sizes = OrderedSet() for x in all_exps(e): if isinstance(x, EStateVar): sizes.add(storage_size(x.e, freebies)) return max_of(*sizes, type=INT)
def exp_wf_nonrecursive(solver, e : Exp, context : Context, pool = RUNTIME_POOL, assumptions : Exp = T): state_vars = OrderedSet(v for v, p in context.vars() if p == STATE_POOL) args = OrderedSet(v for v, p in context.vars() if p == RUNTIME_POOL) assumptions = EAll([assumptions, context.path_condition()]) h = extension_handler(type(e)) if h is not None: msg = h.check_wf(e, state_vars=state_vars, args=args, pool=pool, assumptions=assumptions, is_valid=solver.valid) if msg is not None: raise ExpIsNotWf(e, e, msg) return at_runtime = pool == RUNTIME_POOL if isinstance(e, EStateVar) and not at_runtime: raise ExpIsNotWf(e, e, "EStateVar in state pool position") if isinstance(e, EStateVar): fvs = free_vars(e.e) if not fvs: raise ExpIsNotWf(e, e, "constant value in state position") bad = [v for v in fvs if v not in state_vars] if bad: raise ExpIsNotWf(e, e, "free non-statevars in state position: {}".format(", ".join(v.id for v in bad))) if (isinstance(e, EDropFront) or isinstance(e, EDropBack)) and not at_runtime: raise ExpIsNotWf(e, e, "EDrop* in state position") if isinstance(e, EFlatMap) and not at_runtime: raise ExpIsNotWf(e, e, "EFlatMap in state position") if not allow_int_arithmetic_state.value and not at_runtime and isinstance(e, EBinOp) and e.type == INT: raise ExpIsNotWf(e, e, "integer arithmetic in state position") # if isinstance(e, EUnaryOp) and e.op == UOp.Distinct and not at_runtime: # raise ExpIsNotWf(e, e, "'distinct' in state position") # if isinstance(e, EMapKeys) and not at_runtime: # raise ExpIsNotWf(e, e, "'mapkeys' in state position") if isinstance(e, EVar): if at_runtime and e in state_vars: raise ExpIsNotWf(e, e, "state var at runtime") elif not at_runtime and e in args: raise ExpIsNotWf(e, e, "arg in state exp") # if is_collection(e.type) and is_collection(e.type.t): # raise ExpIsNotWf(e, e, "collection of collection") if is_collection(e.type) and not is_scalar(e.type.t): raise ExpIsNotWf(e, e, "collection of nonscalar") if isinstance(e.type, TMap) and not is_scalar(e.type.k): raise ExpIsNotWf(e, e, "bad key type {}".format(pprint(e.type.k))) if isinstance(e.type, TMap) and isinstance(e.type.v, TMap): raise ExpIsNotWf(e, e, "map to map") # This check is probably a bad idea: whether `the` is legal may depend on # the contex that the expression is embedded within, so we can't skip it # during synthesis just because it looks invalid now. # if isinstance(e, EUnaryOp) and e.op == UOp.The: # len = EUnaryOp(UOp.Length, e.e).with_type(INT) # if not valid(EImplies(assumptions, EBinOp(len, "<=", ENum(1).with_type(INT)).with_type(BOOL))): # raise ExpIsNotWf(e, e, "illegal application of 'the': could have >1 elems") if not at_runtime and isinstance(e, EBinOp) and e.op == "-" and is_collection(e.type): raise ExpIsNotWf(e, e, "collection subtraction in state position") if not at_runtime and isinstance(e, ESingleton): raise ExpIsNotWf(e, e, "singleton in state position") # if not at_runtime and isinstance(e, ENum) and e.val != 0 and e.type == INT: # raise ExpIsNotWf(e, e, "nonzero integer constant in state position") if not allow_conditional_state.value and not at_runtime and isinstance(e, ECond): raise ExpIsNotWf(e, e, "conditional in state position") if isinstance(e, EMakeMap2) and isinstance(e.e, EEmptyList): raise ExpIsNotWf(e, e, "trivially empty map") if do_expensive_checks.value and not at_runtime and isinstance(e, EFilter): # catch "peels": removal of zero or one elements if solver.valid(EImplies(assumptions, ELe(ELen(EFilter(e.e, ELambda(e.p.arg, ENot(e.p.body))).with_type(e.type)), ONE))): raise ExpIsNotWf(e, e, "filter is a peel") if do_expensive_checks.value and not at_runtime and isinstance(e, EMakeMap2) and is_collection(e.type.v): all_collections = [sv for sv in state_vars if is_collection(sv.type)] total_size = ENum(0).with_type(INT) for c in all_collections: total_size = EBinOp(total_size, "+", EUnaryOp(UOp.Length, c).with_type(INT)).with_type(INT) my_size = EUnaryOp(UOp.Length, EFlatMap(EUnaryOp(UOp.Distinct, e.e).with_type(e.e.type), e.value).with_type(e.type.v)).with_type(INT) s = EImplies( assumptions, EBinOp(total_size, ">=", my_size).with_type(BOOL)) if not solver.valid(s): # from cozy.evaluation import eval # from cozy.solver import satisfy # model = satisfy(EAll([assumptions, EBinOp(total_size, "<", my_size).with_type(BOOL)]), collection_depth=3, validate_model=True) # assert model is not None # raise ExpIsNotWf(e, e, "non-polynomial-sized map ({}); total_size={}, this_size={}".format(model, eval(total_size, model), eval(my_size, model))) raise ExpIsNotWf(e, e, "non-polynomial-sized map")
def map_accelerate(e, context): with task("map_accelerate", size=e.size()): if is_constant_time(e): event("skipping map lookup inference for constant-time exp: {}". format(pprint(e))) return @lru_cache() def make_binder(t): return fresh_var(t, hint="key") args = OrderedSet(v for (v, p) in context.vars() if p == RUNTIME_POOL) possible_keys = {} # type -> [exp] i = 0 stk = [e] while stk: event("exp {} / {}".format(i, e.size())) i += 1 arg = stk.pop() if isinstance(arg, tuple): stk.extend(arg) continue if not isinstance(arg, Exp): continue if isinstance(arg, ELambda): stk.append(arg.body) continue if context.legal_for(free_vars(arg)): # all the work happens here binder = make_binder(arg.type) value = replace( e, arg, binder, match=lambda e1, e2: type(e1) == type(e2) and e1.type == e2 .type and alpha_equivalent(e1, e2)) value = strip_EStateVar(value) # print(" ----> {}".format(pprint(value))) if any(v in args for v in free_vars(value)): event("not all args were eliminated") else: if arg.type not in possible_keys: l = [ reachable_values_of_type(sv, arg.type) for (sv, p) in context.vars() if p == STATE_POOL ] l = OrderedSet(x for x in l if not isinstance(x, EEmptyList)) possible_keys[arg.type] = l for keys in possible_keys[arg.type]: # print("reachable values of type {}: {}".format(pprint(arg.type), pprint(keys))) # for v in state_vars: # print(" {} : {}".format(pprint(v), pprint(v.type))) m = EMakeMap2(keys, ELambda(binder, value)).with_type( TMap(arg.type, e.type)) assert not any( v in args for v in free_vars(m)), "oops! {}; args={}".format( pprint(m), ", ".join(pprint(a) for a in args)) yield (m, STATE_POOL) mg = EMapGet(EStateVar(m).with_type(m.type), arg).with_type(e.type) # print(pprint(mg)) # mg._tag = True yield (mg, RUNTIME_POOL) if isinstance(arg, EStateVar): # do not visit state expressions continue num_with_args = 0 stk2 = list(arg.children()) while stk2: child = stk2.pop() if isinstance(child, tuple): stk.extend(child) continue if not isinstance(child, Exp): continue fvs = free_vars(child) if fvs & args: num_with_args += 1 if num_with_args >= 2: break if num_with_args < 2: stk.extend(arg.children()) else: event("refusing to visit children of {}".format(pprint(arg)))
def test_fvs(self): e = EBinOp(EMapGet(EStateVar(EMakeMap2(EVar('l').with_type(TBag(INT)), ELambda(EVar('_var111').with_type(INT), EBinOp(EVar('_var111').with_type(INT), 'in', EVar('l').with_type(TBag(INT))).with_type(BOOL))).with_type(TMap(INT, BOOL))).with_type(TMap(INT, BOOL)), EVar('n').with_type(INT)).with_type(BOOL), '==', EBinOp(EVar('_var111').with_type(INT), 'in', EVar('l').with_type(TBag(INT))).with_type(BOOL)).with_type(BOOL) print(pprint(e)) print(free_vars(e)) assert free_vars(e) == OrderedSet([EVar('l').with_type(TBag(INT)), EVar('n').with_type(INT), EVar('_var111').with_type(INT)])
def good_idea(solver, e: Exp, context: Context, pool=RUNTIME_POOL, assumptions: Exp = T) -> bool: """Heuristic filter to ignore expressions that are almost certainly useless.""" state_vars = OrderedSet(v for v, p in context.vars() if p == STATE_POOL) args = OrderedSet(v for v, p in context.vars() if p == RUNTIME_POOL) assumptions = EAll([assumptions, context.path_condition()]) at_runtime = pool == RUNTIME_POOL if isinstance(e, EStateVar) and not free_vars(e.e): return No("constant value in state position") if (isinstance(e, EDropFront) or isinstance(e, EDropBack)) and not at_runtime: return No("EDrop* in state position") if not allow_big_sets.value and isinstance(e, EFlatMap) and not at_runtime: return No("EFlatMap in state position") if not allow_int_arithmetic_state.value and not at_runtime and isinstance( e, EBinOp) and e.type == INT: return No("integer arithmetic in state position") if is_collection(e.type) and not is_scalar(e.type.t): return No("collection of nonscalar") if isinstance(e.type, TMap) and not is_scalar(e.type.k): return No("bad key type {}".format(pprint(e.type.k))) if isinstance(e.type, TMap) and isinstance(e.type.v, TMap): return No("map to map") # This check is probably a bad idea: whether `the` is legal may depend on # the contex that the expression is embedded within, so we can't skip it # during synthesis just because it looks invalid now. # if isinstance(e, EUnaryOp) and e.op == UOp.The: # len = EUnaryOp(UOp.Length, e.e).with_type(INT) # if not valid(EImplies(assumptions, EBinOp(len, "<=", ENum(1).with_type(INT)).with_type(BOOL))): # return No("illegal application of 'the': could have >1 elems") if not at_runtime and isinstance( e, EBinOp) and e.op == "-" and is_collection(e.type): return No("collection subtraction in state position") # if not at_runtime and isinstance(e, ESingleton): # return No("singleton in state position") # if not at_runtime and isinstance(e, ENum) and e.val != 0 and e.type == INT: # return No("nonzero integer constant in state position") if at_runtime and isinstance(e, EStateVar) and isinstance( e.e, EBinOp) and is_scalar(e.e.e1.type) and is_scalar(e.e.e2.type): return No("constant-time binary operator in state position") if not allow_conditional_state.value and not at_runtime and isinstance( e, ECond): return No("conditional in state position") if isinstance(e, EMakeMap2) and isinstance(e.e, EEmptyList): return No("trivially empty map") if not allow_peels.value and not at_runtime and isinstance(e, EFilter): # catch "peels": removal of zero or one elements if solver.valid( EImplies( assumptions, ELe( ELen( EFilter(e.e, ELambda(e.p.arg, ENot( e.p.body))).with_type(e.type)), ONE))): return No("filter is a peel") if not allow_big_maps.value and not at_runtime and isinstance( e, EMakeMap2) and is_collection(e.type.v): all_collections = [sv for sv in state_vars if is_collection(sv.type)] total_size = ENum(0).with_type(INT) for c in all_collections: total_size = EBinOp(total_size, "+", EUnaryOp(UOp.Length, c).with_type(INT)).with_type(INT) my_size = EUnaryOp( UOp.Length, EFlatMap(EUnaryOp(UOp.Distinct, e.e).with_type(e.e.type), e.value).with_type(e.type.v)).with_type(INT) s = EImplies(assumptions, EBinOp(total_size, ">=", my_size).with_type(BOOL)) if not solver.valid(s): # from cozy.evaluation import eval # from cozy.solver import satisfy # model = satisfy(EAll([assumptions, EBinOp(total_size, "<", my_size).with_type(BOOL)]), collection_depth=3, validate_model=True) # assert model is not None # return No("non-polynomial-sized map ({}); total_size={}, this_size={}".format(model, eval(total_size, model), eval(my_size, model))) return No("non-polynomial-sized map") return True
def possibly_useful_nonrecursive( solver, e: Exp, context: Context, pool=RUNTIME_POOL, assumptions: Exp = ETRUE, ops: [Op] = ()) -> bool: """Heuristic filter to ignore expressions that are almost certainly useless.""" state_vars = OrderedSet(v for v, p in context.vars() if p == STATE_POOL) args = OrderedSet(v for v, p in context.vars() if p == RUNTIME_POOL) assumptions = EAll([assumptions, context.path_condition()]) at_runtime = pool == RUNTIME_POOL h = extension_handler(type(e)) if h is not None: res = h.possibly_useful(e, context, pool, assumptions, ops, solver) if not res: return res if isinstance(e, EStateVar) and not free_vars(e.e): return No("constant value in state position") if (isinstance(e, EDropFront) or isinstance(e, EDropBack)) and not at_runtime: return No("EDrop* in state position") if not allow_big_sets.value and isinstance(e, EFlatMap) and not at_runtime: return No("EFlatMap in state position") if not allow_int_arithmetic_state.value and not at_runtime and isinstance( e, EBinOp) and e.type == INT: return No("integer arithmetic in state position") if is_collection(e.type) and not is_scalar(e.type.elem_type): return No("collection of nonscalar: e {}\n elem_type: {}\n".format( e, e.type.elem_type)) if isinstance(e.type, TMap) and not is_scalar(e.type.k): return No("bad key type {}".format(pprint(e.type.k))) if isinstance(e.type, TMap) and isinstance(e.type.v, TMap): return No("map to map") # This check is probably a bad idea: whether `the` is legal may depend on # the contex that the expression is embedded within, so we can't skip it # during synthesis just because it looks invalid now. # if isinstance(e, EUnaryOp) and e.op == UOp.The: # len = EUnaryOp(UOp.Length, e.e).with_type(INT) # if not valid(EImplies(assumptions, EBinOp(len, "<=", ENum(1).with_type(INT)).with_type(BOOL))): # return No("illegal application of 'the': could have >1 elems") if not at_runtime and isinstance( e, EBinOp) and e.op == "-" and is_collection(e.type): return No("collection subtraction in state position") # if not at_runtime and isinstance(e, ESingleton): # return No("singleton in state position") if not allow_nonzero_state_constants.value and not at_runtime and isinstance( e, ENum) and e.val != 0: return No("nonzero integer constant in state position") if not allow_binop_state.value and at_runtime and isinstance( e, EStateVar) and isinstance(e.e, EBinOp) and is_scalar( e.e.e1.type) and is_scalar(e.e.e2.type): return No( "constant-time binary operator {!r} in state position".format( e.e.op)) if not allow_conditional_state.value and not at_runtime and isinstance( e, ECond): return No("conditional in state position") if isinstance(e, EMakeMap2) and isinstance(e.e, EEmptyList): return No("trivially empty map") if isinstance(e, EMakeMap2) and isinstance(e.e, ESingleton): return No("really tiny map") if not at_runtime and (isinstance(e, EArgMin) or isinstance(e, EArgMax)): # Cozy has no way to efficiently implement mins/maxes when more than # one element may leave the collection. from cozy.state_maintenance import mutate for op in ops: elems = e.e elems_prime = mutate(elems, op.body) formula = EAll([assumptions] + list(op.assumptions) + [ EGt( ELen( EBinOp(elems, "-", elems_prime).with_type(elems.type)), ONE) ]) if solver.satisfiable(formula): return No( "more than one element might be removed during {}".format( op.name)) if not allow_peels.value and not at_runtime and isinstance(e, EFilter): # catch "peels": removal of zero or one elements if solver.valid( EImplies( assumptions, ELe( ELen( EFilter( e.e, ELambda(e.predicate.arg, ENot( e.predicate.body))).with_type(e.type)), ONE))): return No("filter is a peel") if not allow_big_maps.value and not at_runtime and isinstance( e, EMakeMap2) and is_collection(e.type.v): all_collections = [sv for sv in state_vars if is_collection(sv.type)] total_size = ENum(0).with_type(INT) for c in all_collections: total_size = EBinOp(total_size, "+", EUnaryOp(UOp.Length, c).with_type(INT)).with_type(INT) my_size = EUnaryOp( UOp.Length, EFlatMap( EUnaryOp(UOp.Distinct, e.e).with_type(e.e.type), e.value_function).with_type(e.type.v)).with_type(INT) s = EImplies(assumptions, EBinOp(total_size, ">=", my_size).with_type(BOOL)) if not solver.valid(s): return No("non-polynomial-sized map") return True
def enumerate_core(self, context: Context, size: int, pool: Pool) -> [Exp]: """ Arguments: conext : a Context object describing the vars in scope size : size to enumerate pool : pool to enumerate Yields all expressions of the given size legal in the given context and pool. """ if size < 0: return if size == 0: for e in LITERALS: yield e all_interesting_types = OrderedSet() for v, _ in context.vars(): all_interesting_types |= all_types(v) for h, _, _ in self.hints: all_interesting_types |= all_types(h) for t in all_interesting_types: l = construct_value(t) if l not in LITERALS: yield l for (v, p) in context.vars(): if p == pool: yield v for (e, ctx, p) in self.hints: if p == pool and ctx.alpha_equivalent(context): yield context.adapt(e, ctx) return yield from self.heuristic_enumeration(context, size, pool) for e in collections(self.enumerate(context, size - 1, pool)): yield EEmptyList().with_type(e.type) if is_numeric(e.type.t): yield EUnaryOp(UOp.Sum, e).with_type(e.type.t) for e in self.enumerate(context, size - 1, pool): yield ESingleton(e).with_type(TBag(e.type)) for e in self.enumerate(context, size - 1, pool): if isinstance(e.type, TRecord): for (f, t) in e.type.fields: yield EGetField(e, f).with_type(t) for e in self.enumerate(context, size - 1, pool): if isinstance(e.type, THandle): yield EGetField(e, "val").with_type(e.type.value_type) for e in self.enumerate(context, size - 1, pool): if isinstance(e.type, TTuple): for n in range(len(e.type.ts)): yield ETupleGet(e, n).with_type(e.type.ts[n]) for e in of_type(self.enumerate(context, size - 1, pool), BOOL): yield EUnaryOp(UOp.Not, e).with_type(BOOL) for e in self.enumerate(context, size - 1, pool): if is_numeric(e.type): yield EUnaryOp("-", e).with_type(e.type) for m in self.enumerate(context, size - 1, pool): if isinstance(m.type, TMap): yield EMapKeys(m).with_type(TBag(m.type.k)) for (sz1, sz2) in pick_to_sum(2, size - 1): for a1 in self.enumerate(context, sz1, pool): t = a1.type if not is_numeric(t): continue for a2 in of_type(self.enumerate(context, sz2, pool), t): yield EBinOp(a1, "+", a2).with_type(t) yield EBinOp(a1, "-", a2).with_type(t) yield EBinOp(a1, ">", a2).with_type(BOOL) yield EBinOp(a1, "<", a2).with_type(BOOL) yield EBinOp(a1, ">=", a2).with_type(BOOL) yield EBinOp(a1, "<=", a2).with_type(BOOL) for a1 in collections(self.enumerate(context, sz1, pool)): for a2 in of_type(self.enumerate(context, sz2, pool), a1.type): yield EBinOp(a1, "+", a2).with_type(a1.type) yield EBinOp(a1, "-", a2).with_type(a1.type) for a2 in of_type(self.enumerate(context, sz2, pool), a1.type.t): yield EBinOp(a2, BOp.In, a1).with_type(BOOL) for a1 in of_type(self.enumerate(context, sz1, pool), BOOL): for a2 in of_type(self.enumerate(context, sz2, pool), BOOL): yield EBinOp(a1, BOp.And, a2).with_type(BOOL) yield EBinOp(a1, BOp.Or, a2).with_type(BOOL) for a1 in self.enumerate(context, sz1, pool): if not isinstance(a1.type, TMap): for a2 in of_type(self.enumerate(context, sz2, pool), a1.type): yield EEq(a1, a2) yield EBinOp(a1, "!=", a2).with_type(BOOL) for m in self.enumerate(context, sz1, pool): if isinstance(m.type, TMap): for k in of_type(self.enumerate(context, sz2, pool), m.type.k): yield EMapGet(m, k).with_type(m.type.v) yield EHasKey(m, k).with_type(BOOL) for l in self.enumerate(context, sz1, pool): if not isinstance(l.type, TList): continue for i in of_type(self.enumerate(context, sz2, pool), INT): yield EListGet(l, i).with_type(l.type.t) for (sz1, sz2, sz3) in pick_to_sum(3, size - 1): for cond in of_type(self.enumerate(context, sz1, pool), BOOL): for then_branch in self.enumerate(context, sz2, pool): for else_branch in of_type( self.enumerate(context, sz2, pool), then_branch.type): yield ECond(cond, then_branch, else_branch).with_type(then_branch.type) for l in self.enumerate(context, sz1, pool): if not isinstance(l.type, TList): continue for st in of_type(self.enumerate(context, sz2, pool), INT): for ed in of_type(self.enumerate(context, sz3, pool), INT): yield EListSlice(l, st, ed).with_type(l.type) for bag in collections(self.enumerate(context, size - 1, pool)): # len of bag count = EUnaryOp(UOp.Length, bag).with_type(INT) yield count # empty? yield EUnaryOp(UOp.Empty, bag).with_type(BOOL) # exists? yield EUnaryOp(UOp.Exists, bag).with_type(BOOL) # singleton? yield EEq(count, ONE) yield EUnaryOp(UOp.The, bag).with_type(bag.type.t) yield EUnaryOp(UOp.Distinct, bag).with_type(bag.type) yield EUnaryOp(UOp.AreUnique, bag).with_type(BOOL) if bag.type.t == BOOL: yield EUnaryOp(UOp.Any, bag).with_type(BOOL) yield EUnaryOp(UOp.All, bag).with_type(BOOL) def build_lambdas(bag, pool, body_size): v = fresh_var(bag.type.t, omit=set(v for v, p in context.vars())) inner_context = UnderBinder(context, v=v, bag=bag, bag_pool=pool) for lam_body in self.enumerate(inner_context, body_size, pool): yield ELambda(v, lam_body) # Let-expressions for (sz1, sz2) in pick_to_sum(2, size - 1): for x in self.enumerate(context, sz1, pool): bag = ESingleton(x).with_type(TBag(x.type)) for lam in build_lambdas(bag, pool, sz2): e = ELet(x, lam).with_type(lam.body.type) # if x == EBinOp(EVar("x"), "+", EVar("x")): # e._tag = True yield e # Iteration for (sz1, sz2) in pick_to_sum(2, size - 1): for bag in collections(self.enumerate(context, sz1, pool)): for lam in build_lambdas(bag, pool, sz2): body_type = lam.body.type yield EMap(bag, lam).with_type(TBag(body_type)) if body_type == BOOL: yield EFilter(bag, lam).with_type(bag.type) if is_numeric(body_type): yield EArgMin(bag, lam).with_type(bag.type.t) yield EArgMax(bag, lam).with_type(bag.type.t) if is_collection(body_type): yield EFlatMap(bag, lam).with_type(TBag(body_type.t)) # Enable use of a state-pool expression at runtime if pool == RUNTIME_POOL: for e in self.enumerate(context, size - 1, STATE_POOL): yield EStateVar(e).with_type(e.type) # Create maps if pool == STATE_POOL: for (sz1, sz2) in pick_to_sum(2, size - 1): for bag in collections(self.enumerate(context, sz1, STATE_POOL)): if not is_scalar(bag.type.t): continue for lam in build_lambdas(bag, STATE_POOL, sz2): t = TMap(bag.type.t, lam.body.type) m = EMakeMap2(bag, lam).with_type(t) yield m
def __init__(self, wrapped_builder, state_vars, args, binders_to_use, assumptions : Exp): self.wrapped_builder = wrapped_builder self.state_vars = OrderedSet(state_vars) self.args = OrderedSet(args) self.binders_to_use = binders_to_use self.assumptions = assumptions
def _enumerate_core(self, context: Context, size: int, pool: Pool) -> [Exp]: """Build new expressions of the given size. Arguments: context : a Context object describing the vars in scope size : size of expressions to enumerate; each expression in the output will have this size pool : pool to enumerate This function is not cached. Clients should call `enumerate` instead. This function tries to be a clean description of the Cozy grammar. It does not concern itself with deduplication (which is handled efficiently by equivalence class deduplication). """ if size < 0: return if size == 0: for e in LITERALS: yield e all_interesting_types = OrderedSet(self.hint_types) for v, _ in context.vars(): all_interesting_types |= all_types(v.type) for t in all_interesting_types: l = construct_value(t) if l not in LITERALS: yield l for (v, p) in context.vars(): if p == pool: yield v for (e, ctx, p) in self.hints: if p == pool: fvs = free_vars(e) if ctx.alpha_equivalent(context.generalize(fvs)): yield context.adapt(e, ctx, e_fvs=fvs) return if not do_enumerate.value: return def build_lambdas(bag, pool, body_size): v = fresh_var(bag.type.elem_type, omit=set(v for v, p in context.vars())) inner_context = UnderBinder(context, v=v, bag=bag, bag_pool=pool) for lam_body in self.enumerate(inner_context, body_size, pool): yield ELambda(v, lam_body) # Load all smaller expressions in this context and pool. # cache[S] contains expressions of size S in this context and pool. cache = [list(self.enumerate(context, sz, pool)) for sz in range(size)] # Enable use of a state-pool expression at runtime if pool == RUNTIME_POOL: for e in self.enumerate(context.root(), size - 1, STATE_POOL): yield EStateVar(e).with_type(e.type) # Arity-1 expressions for e in cache[size - 1]: if is_collection(e.type): elem_type = e.type.elem_type # This method of generating EEmptyList() ensures that we visit # empty collections of all possible types. yield EEmptyList().with_type(e.type) if is_numeric(elem_type): yield EUnaryOp(UOp.Sum, e).with_type(elem_type) yield EUnaryOp(UOp.Length, e).with_type(INT) yield EUnaryOp(UOp.Empty, e).with_type(BOOL) yield EUnaryOp(UOp.Exists, e).with_type(BOOL) yield EUnaryOp(UOp.The, e).with_type(elem_type) yield EUnaryOp(UOp.Distinct, e).with_type(e.type) yield EUnaryOp(UOp.AreUnique, e).with_type(BOOL) if elem_type == BOOL: yield EUnaryOp(UOp.Any, e).with_type(BOOL) yield EUnaryOp(UOp.All, e).with_type(BOOL) yield ESingleton(e).with_type(TBag(e.type)) if isinstance(e.type, TRecord): for (f, t) in e.type.fields: yield EGetField(e, f).with_type(t) if isinstance(e.type, THandle): yield EGetField(e, "val").with_type(e.type.value_type) if isinstance(e.type, TTuple): for n in range(len(e.type.ts)): yield ETupleGet(e, n).with_type(e.type.ts[n]) if e.type == BOOL: yield EUnaryOp(UOp.Not, e).with_type(BOOL) if is_numeric(e.type): yield EUnaryOp("-", e).with_type(e.type) if isinstance(e.type, TMap): yield EMapKeys(e).with_type(TBag(e.type.k)) # Arity-2 expressions for (sz1, sz2) in pick_to_sum(2, size - 1): # sz1 + sz2 = size - 1 for e1 in cache[sz1]: t = e1.type if is_numeric(t): for a2 in of_type(cache[sz2], t): yield EBinOp(e1, "+", a2).with_type(t) yield EBinOp(e1, "-", a2).with_type(t) if is_ordered(t): for a2 in of_type(cache[sz2], t): yield EBinOp(e1, ">", a2).with_type(BOOL) yield EBinOp(e1, "<", a2).with_type(BOOL) yield EBinOp(e1, ">=", a2).with_type(BOOL) yield EBinOp(e1, "<=", a2).with_type(BOOL) if t == BOOL: for a2 in of_type(cache[sz2], BOOL): yield EBinOp(e1, BOp.And, a2).with_type(BOOL) yield EBinOp(e1, BOp.Or, a2).with_type(BOOL) # Cozy supports the implication operator "=>", but this # function does not enumerate it because # - (a => b) is equivalent to ((not a) or b) # - there isn't an implication operator in any of our # current target languages, so we would need to # desugar it to ((not a) or b) anyway. if not isinstance(t, TMap): for a2 in of_type(cache[sz2], t): yield EEq(e1, a2) yield EBinOp(e1, "!=", a2).with_type(BOOL) if isinstance(t, TMap): for k in of_type(cache[sz2], t.k): yield EMapGet(e1, k).with_type(t.v) yield EHasKey(e1, k).with_type(BOOL) if isinstance(t, TList): for i in of_type(cache[sz2], INT): yield EListGet(e1, i).with_type(e1.type.elem_type) if is_collection(t): elem_type = t.elem_type for e2 in of_type(cache[sz2], t): yield EBinOp(e1, "+", e2).with_type(t) yield EBinOp(e1, "-", e2).with_type(t) for e2 in of_type(cache[sz2], elem_type): yield EBinOp(e2, BOp.In, e1).with_type(BOOL) for f in build_lambdas(e1, pool, sz2): body_type = f.body.type yield EMap(e1, f).with_type(TBag(body_type)) if body_type == BOOL: yield EFilter(e1, f).with_type(t) if is_numeric(body_type): yield EArgMin(e1, f).with_type(elem_type) yield EArgMax(e1, f).with_type(elem_type) if is_collection(body_type): yield EFlatMap(e1, f).with_type( TBag(body_type.elem_type)) if pool == STATE_POOL and is_hashable(elem_type): yield EMakeMap2(e1, f).with_type( TMap(elem_type, body_type)) e1_singleton = ESingleton(e1).with_type(TBag(e1.type)) for f in build_lambdas(e1_singleton, pool, sz2): yield ELet(e1, f).with_type(f.body.type) # Arity-3 expressions for (sz1, sz2, sz3) in pick_to_sum(3, size - 1): # sz1 + sz2 + sz3 = size - 1 for e1 in cache[sz1]: if e1.type == BOOL: cond = e1 for then_branch in cache[sz2]: for else_branch in of_type(cache[sz3], then_branch.type): yield ECond(cond, then_branch, else_branch).with_type( then_branch.type) if isinstance(e1.type, TList): for start in of_type(cache[sz2], INT): for end in of_type(cache[sz3], INT): yield EListSlice(e1, start, end).with_type(e1.type) # It is not necessary to create slice expressions of # the form a[:i] or a[i:]. Those are desugared # after parsing to a[0:i] and a[i:len(a)] # respectively, and Cozy is perfectly capable of # discovering these expanded forms as well. for h in all_extension_handlers(): yield from h.enumerate(context, size, pool, self.enumerate, build_lambdas)
def vars(self): return OrderedSet(itertools.chain( [(v, STATE_POOL) for v in self.state_vars], [(v, RUNTIME_POOL) for v in self.args]))
def __init__(self, state_vars : [Exp], args : [Exp], funcs : {str:TFunc} = None): self.state_vars = OrderedSet(state_vars) self.args = OrderedSet(args) self.functions = OrderedDict(funcs or ()) assert not (self.state_vars & self.args)