def _simplify(cls, invs, is_conj, use_reals): assert invs, invs st = time() eqts, eqts_largecoefs, octs, mps, preposts, falseinvs = \ cls.classify(invs) def mysorted(ps): return sorted(ps, key=lambda p: len(Miscs.get_vars(p.inv))) eqts = mysorted(eqts + eqts_largecoefs) octs = mysorted(octs) mps = mysorted(mps) myinvs = eqts + falseinvs + preposts + octs + mps myinvs_exprs = [inv.expr(use_reals) for inv in myinvs] def _imply(js, i): iexpr = myinvs_exprs[i] # don't consider/remove equality if iexpr.decl().kind() == z3.Z3_OP_EQ: ret = False else: jexprs = [myinvs_exprs[j] for j in js] ret = Z3._imply(jexprs, iexpr, is_conj) # if ret: # print '{} => {}'.format(jexprs, iexpr) return ret results = Miscs.simplify_idxs(list(range(len(myinvs))), _imply) results = [myinvs[i] for i in results] Miscs.show_removed('_simplify', len(invs), len(results), time() - st) return results
def filter_terms(terms, inps): assert isinstance(inps, set) and \ all(isinstance(s, str) for s in inps), inps if not inps: mlog.warning("Have not tested case with no inps") excludes = set() for term in terms: if isinstance(term, data.poly.mp.MP): a_symbs = list(map(str, Miscs.get_vars(term.a))) b_symbs = list(map(str, Miscs.get_vars(term.b))) inp_in_a = any(s in inps for s in a_symbs) inp_in_b = any(s in inps for s in b_symbs) if ((inp_in_a and inp_in_b) or (not inp_in_a and not inp_in_b)): excludes.add(term) continue t_symbs = set(a_symbs + b_symbs) else: t_symbs = set(map(str, term.symbols)) if len(t_symbs) <= 1: # ok for finding bound of single input val continue if (inps.issuperset(t_symbs) or all(s not in inps for s in t_symbs)): excludes.add(term) new_terms = [term for term in terms if term not in excludes] return new_terms
def _get_init_traces(self, loc, deg, traces, inps, rate): "Initial loop to obtain (random) traces to bootstrap eqt solving" assert deg >= 1, deg assert isinstance(rate, float) and rate >= 0.1, rate if self.use_rand_init: whileF, whileFName = self._while_rand, "random" else: whileF, whileFName = self._while_symstates, "symstates" mlog.debug( "{}: gen init inps using {} (curr inps {}, traces {})".format( loc, whileFName, len(inps), len(traces))) terms, template, uks, n_eqts_needed = Miscs.init_terms( self.inv_decls[loc].names, deg, rate) exprs = whileF(loc, template, n_eqts_needed, inps, traces) # if cannot generate sufficient traces, adjust degree while (not exprs): if deg < 2: return # cannot generate enough traces deg = deg - 1 mlog.info( "Reduce polynomial degree to {}, terms {}, uks {}".format( deg, len(terms), len(uks))) terms, template, uks, n_eqts_needed = Miscs.init_terms( self.inv_decls[loc].names, deg, rate) exprs = whileF(loc, template, n_eqts_needed, inps, traces) return template, uks, exprs
def infer_eqts(self, maxdeg, symbols, traces): auto_deg = self.get_auto_deg(maxdeg) terms, template, uks, n_eqts_needed = Miscs.init_terms( symbols.names, auto_deg, settings.EQT_RATE) exprs = list(traces.instantiate(template, n_eqts_needed)) eqts = Miscs.solve_eqts(exprs, uks, template) import data.inv.eqt return [data.inv.eqt.Eqt(eqt) for eqt in eqts]
def _infer(self, loc, template, uks, exprs, dtraces, inps): assert isinstance(loc, str) and loc, loc assert Miscs.is_expr(template), template assert isinstance(uks, list), uks assert isinstance(exprs, set) and exprs, exprs assert isinstance(dtraces, DTraces) and dtraces, dtraces assert isinstance(inps, Inps) and inps, inps cache = set() eqts = set() # results exprs = list(exprs) new_cexs = [] curIter = 0 while True: curIter += 1 mlog.debug("{}, iter {} infer using {} exprs".format( loc, curIter, len(exprs))) new_eqts = Miscs.solve_eqts(exprs, uks, template) unchecks = [eqt for eqt in new_eqts if eqt not in cache] if not unchecks: mlog.debug("{}: no new results -- break".format(loc)) break mlog.debug('{}: {} candidates:\n{}'.format( loc, len(new_eqts), '\n'.join(map(str, new_eqts)))) mlog.debug("{}: check {} unchecked ({} candidates)".format( loc, len(unchecks), len(new_eqts))) dinvs = DInvs.mk(loc, Invs(list(map(data.inv.eqt.Eqt, unchecks)))) cexs, dinvs = self.check(dinvs, None) if cexs: new_cexs.append(cexs) [eqts.add(inv) for inv in dinvs[loc] if not inv.is_disproved] [cache.add(inv.inv) for inv in dinvs[loc] if inv.stat is not None] if loc not in cexs: mlog.debug("{}: no disproved candidates -- break".format(loc)) break cexs = Traces.extract(cexs[loc]) cexs = cexs.padzeros(set(self.inv_decls[loc].names)) exprs_ = cexs.instantiate(template, None) mlog.debug("{}: {} new cex exprs".format(loc, len(exprs_))) exprs.extend(exprs_) return eqts, new_cexs
def test(self, dtraces): # assert isinstance(dtraces, DTraces) assert self.siz, self st = time() tasks = [loc for loc in self if self[loc]] def f(tasks): return [(loc, self[loc].test(dtraces[loc])) for loc in tasks] wrs = Miscs.run_mp("test_dinvs", tasks, f) dinvs = DInvs([(loc, invs) for loc, invs in wrs if invs]) Miscs.show_removed("test_dinvs", self.siz, dinvs.siz, time() - st) return dinvs
def simplify(self, use_reals): assert isinstance(use_reals, bool), use_reals assert (self.siz), self st = time() def f(tasks): return [(loc, self[loc].simplify(use_reals)) for loc in tasks] wrs = Miscs.run_mp('simplify', list(self), f) dinvs = self.__class__((loc, invs) for loc, invs in wrs if invs) Miscs.show_removed('simplify', self.siz, dinvs.siz, time() - st) return dinvs
def gen(self, deg, traces, inps): assert deg >= 1, deg assert isinstance(traces, DTraces) and traces, traces assert isinstance(inps, Inps), inps locs = traces.keys() # first obtain enough traces tasks = [(loc, self._get_init_traces(loc, deg, traces, inps, settings.EQT_RATE)) for loc in locs] tasks = [(loc, tcs) for loc, tcs in tasks if tcs] # then solve/prove in parallel def f(tasks): return [(loc, self._infer(loc, template, uks, exprs, traces, inps)) for loc, (template, uks, exprs) in tasks] wrs = Miscs.run_mp('find eqts', tasks, f) # put results together dinvs = DInvs() for loc, (eqts, cexs) in wrs: new_inps = inps.merge(cexs, self.inp_decls.names) mlog.debug("{}: got {} eqts, {} new inps".format( loc, len(eqts), len(new_inps))) if eqts: mlog.debug('\n'.join(map(str, eqts))) dinvs[loc] = Invs(eqts) return dinvs
def myeval(self, expr, pred=None): assert Miscs.is_expr(expr), expr if pred is None: return [trace.myeval(expr) for trace in self] else: return any(pred(trace.myeval(expr)) for trace in self)
def get_preconds(cls, symbols, term_siz): """ sage: x,y,z = sage.all.var('x y z') #doctest: +NORMALIZE_WHITESPACE sage: sorted(CegirPrePosts._preconds([x,y], 2), key=str) [-x + y < 0, -x + y <= 0, -x - y < 0, -x - y <= 0, -x < 0, -x <= 0, -y < 0, -y <= 0, x + y < 0, x + y <= 0, x - y < 0, x - y <= 0, x < 0, x <= 0, x == 0, y < 0, y <= 0, y == 0] """ t1 = [Eqt(t == 0) for t in symbols] # M=0, N=0 ts = Miscs.get_terms_fixed_coefs(symbols, term_siz, settings.ICOEFS) t2 = [Oct(t < 0) for t in ts] # +/M+/-N >0 t3 = [Oct(t <= 0) for t in ts] # +/M+/-N >=0 return t1 + t2 + t3
def simplify(self, use_reals): assert isinstance(use_reals, bool), use_reals eqts, eqts_largecoefs, octs, mps, preposts, falseinvs = \ self.classify(self) assert not falseinvs, falseinvs non_mps = eqts + preposts + octs if non_mps and len(mps) >= 2: # parallelizing simplifying mps non_mps_exprs = [e.expr(use_reals) for e in non_mps] conj = z3.And(non_mps_exprs) def f(mps): return [ mp for mp in mps if not Z3._imply(conj, mp.expr(use_reals)) ] wrs = Miscs.run_mp("simplifying {} mps".format(len(mps)), mps, f) mps = [mp for mp in wrs] rs = non_mps + mps if rs: is_conj = True rs = self._simplify(rs, is_conj, use_reals) return self.__class__(rs + eqts_largecoefs)
def prove_reach(self, vs, input): # @timeit def f(task): vid, vld_cls = task vld = vld_cls(self.tmpdir) r, cex = vld.prove_reach(vs, input) if r is None: return None else: return vid, (r, cex) wrs = Miscs.run_mp_ex( "prove_reach", [(settings.CPAchecker.CPA_SHORT_NAME, CPAchecker), (settings.Ultimate.UAUTOMIZER_SHORT_NAME, UAutomizer), (settings.Ultimate.UTAIPAN_SHORT_NAME, UTaipan)], f, get_fst_res=True) mlog.debug('wrs: {}'.format(wrs)) if wrs: vid, r = wrs[0] mlog.debug('Got result firstly from {}'.format(vid)) return r else: return None, None
def get_postconds(cls, eqt): assert Miscs.is_expr(eqt), eqt assert eqt.operator() == operator.eq, eqt # tCtr symbols = [ s for s in Miscs.get_vars(eqt) if settings.CTR_VAR in str(s) ] if not symbols: return assert len(symbols) == 1, \ "should only have 1 tCtr symbol: {}, {}".format( symbols, settings.CTR_VAR) postconds = sage.all.solve(eqt, symbols[0]) return postconds if len(postconds) >= 1 else None
def test_single_trace(self, trace): assert isinstance(trace, data.traces.Trace), trace # temp fix: disable repeating rational when testing equality if (any(not x.is_integer() and Miscs.is_repeating_rational(x) for x in trace.vs)): mlog.debug("skip trace with repeating rational {}".format(self)) return True return super().test_single_trace(trace)
def get_terms(self, symbols): terms = [] if settings.DO_IEQS: oct_siz = 2 terms_ieqs = Miscs.get_terms_fixed_coefs(symbols, oct_siz) terms_ieqs = [data.poly.base.GeneralPoly(t) for t in terms_ieqs] mlog.debug("{} terms for Ieqs".format(len(terms_ieqs))) terms.extend(terms_ieqs) if settings.DO_MINMAXPLUS: terms_u = data.poly.mp.MP.get_terms(symbols) terms_u_no_octs = [(a, b) for a, b in terms_u if len(b) >= 2] if settings.DO_IEQS: # ignore oct invs terms_u = terms_u_no_octs def _get_terms(terms_u, is_max): terms_l = [(b, a) for a, b in terms_u] terms = terms_u + terms_l terms = [data.poly.mp.MP(a, b, is_max) for a, b in terms] return terms terms_max = _get_terms(terms_u, is_max=True) terms_min = _get_terms(terms_u_no_octs, is_max=False) terms_mp = terms_min + terms_max terms.extend(terms_mp) mlog.debug("{} terms for MP".format(len(terms_mp))) print(terms_mp) if settings.DO_TERM_FILTER: st = time() new_terms = self.filter_terms(terms, set(self.prog.inp_decls.names)) Miscs.show_removed('term filter', len(terms), len(new_terms), time() - st) return new_terms else: return terms
def instantiate(self, term, ntraces): assert Miscs.is_expr(term), term assert ntraces is None or ntraces >= 1, ntraces if ntraces is None: for t in self.mydicts: exprs = set(term.subs(t) for t in self.mydicts) else: ntracesExtra = ntraces * settings.TRACE_MULTIPLIER exprs = set() for t in self.mydicts: expr = term.subs(t) if expr not in exprs: exprs.add(expr) if len(exprs) >= ntracesExtra: break # instead of doing this, can find out the # 0's in traces # the more 0's , the better exprs = sorted(exprs, key=lambda expr: len(Miscs.get_vars(expr))) exprs = set(exprs[:ntraces]) return exprs
def get_conj_preconds(self, loc, preconds, postcond_expr): """ preconds => post can be strengthened by removing some preconds e.g., a&b => post is stronger than a&b&c => post """ assert all(isinstance(p, Inv) for p in preconds), preconds assert z3.is_expr(postcond_expr), postcond_expr if not preconds: return [] preconds = sorted(preconds, key=lambda p: len(Miscs.get_vars(p.inv))) preconds_exprs = [pc.expr(self.use_reals) for pc in preconds] if not self.check(preconds_exprs, postcond_expr, loc): return [] def _imply(js, _): jexprs = [preconds_exprs[j] for j in js] return self.check(jexprs, postcond_expr, loc) results = Miscs.simplify_idxs(list(range(len(preconds))), _imply) results = [preconds[i] for i in results] return results
def gen(self, traces, inps): assert isinstance(traces, data.traces.DTraces) and traces, traces assert isinstance(inps, data.traces.Inps), inps statss = [] locs = traces.keys() termss = [ self.get_terms(self.inv_decls[loc].sageExprs) for loc in locs ] mlog.debug("check upperbounds for {} terms at {} locs".format( sum(map(len, termss)), len(locs))) maxV = settings.OCT_MAX_V minV = -1 * maxV refs = { loc: {self.mk_le(t, maxV): t for t in terms} for loc, terms in zip(locs, termss) } ieqs = data.inv.invs.DInvs([(loc, data.inv.invs.Invs(refs[loc].keys())) for loc in refs]) cexs, ieqs, stats = self.check(ieqs, inps=None) statss.extend(stats) if cexs: cexs_inps = inps.merge(cexs, self.inp_decls.names) if cexs_inps: self.get_traces(cexs_inps, traces) ieqs = ieqs.remove_disproved() tasks = [(loc, refs[loc][mp]) for loc in ieqs for mp in ieqs[loc]] mlog.debug("{} locs: infer upperbounds for {} terms".format( len(locs), len(tasks))) def f(tasks): return [(loc, self.gc(loc, term, minV, maxV, traces)) for loc, term in tasks] wrs = Miscs.run_mp('guesscheck', tasks, f) dinvs = data.inv.invs.DInvs() for loc, (inv, stats) in wrs: statss.extend(stats) if inv: dinvs.setdefault(loc, data.inv.invs.Invs()).add(inv) return dinvs, statss
def start(self, seed, maxdeg): assert maxdeg is None or maxdeg >= 1, maxdeg super().start(seed, maxdeg) st = time.time() tasks = [] for loc in self.dtraces: symbols = self.inv_decls[loc] traces = self.dtraces[loc] if settings.DO_EQTS: def _f(): return self.infer_eqts(maxdeg, symbols, traces) tasks.append((loc, _f)) if settings.DO_IEQS: def _f(): return self.infer_ieqs(symbols, traces) tasks.append((loc, _f)) def f(tasks): rs = [(loc, _f()) for loc, _f in tasks] return rs wrs = Miscs.run_mp("(pure) dynamic inference", tasks, f) dinvs = DInvs() for loc, invs in wrs: for inv in invs: dinvs.add(loc, inv) try: new_traces = self.dtraces.merge(self.test_dtraces) mlog.debug('added {} test traces'.format(new_traces.siz)) except AttributeError: # no test traces pass dinvs = self.sanitize(dinvs, self.dtraces) print(dinvs)
def test(self, traces): # assert isinstance(traces, Traces) assert (self), self def f(tasks): return [(inv, inv.test(traces)) for inv in tasks] wrs = Miscs.run_mp("test", list(self), f) myinvs = set() for inv, passed in wrs: if passed: myinvs.add(inv) else: mlog.debug("remove {}".format(inv)) invs = self.__class__(myinvs) return invs
def _get_traces_mp(self, inps): """ run program on inps and obtain traces in parallel return {inp: traces} """ assert isinstance(inps, data.traces.Inps), inps tasks = [inp for inp in inps if inp not in self._cache] def f(tasks): return [(inp, self._get_traces(inp)) for inp in tasks] wrs = Miscs.run_mp("get traces", tasks, f) for inp, traces in wrs: assert inp not in self._cache self._cache[inp] = traces return {inp: self._cache[inp] for inp in inps}
def classify(cls, invs): eqts, eqts_largecoefs, octs, mps, preposts, falseinvs = [], [], [], [], [], [] for inv in invs: if isinstance(inv, data.inv.eqt.Eqt): if len(Miscs.get_coefs(inv.inv)) > 10: eqts_largecoefs.append(inv) else: eqts.append(inv) elif isinstance(inv, data.inv.oct.Oct): octs.append(inv) elif isinstance(inv, data.inv.mp.MP): mps.append(inv) elif isinstance(inv, data.inv.prepost.PrePost): preposts.append(inv) else: assert isinstance(inv, data.inv.invs.FalseInv), inv falseinvs.append(inv) return eqts, eqts_largecoefs, octs, mps, preposts, falseinvs
def infer_ieqs(self, symbols, traces): maxV = settings.IUPPER minV = -1 * maxV terms = Miscs.get_terms_fixed_coefs( symbols.sageExprs, settings.ITERMS, settings.ICOEFS, ) ieqs = [] for t in terms: upperbound = max(traces.myeval(t)) if upperbound > maxV or upperbound < minV: continue ieqs.append(t <= upperbound) import data.inv.oct ieqs = [data.inv.oct.Oct(ieq) for ieq in ieqs] return ieqs
def _merge_traces(): # def f(task): # inp, l = task # # vtrace1: 8460 16 0 1 16 8460 # parts = l.split(':') # assert len(parts) == 2, parts # loc, tracevals = parts[0], parts[1] # loc = loc.strip() # vtrace1 # ss = inv_decls[loc].names # vs = tracevals.strip().split() # trace = Trace.parse(ss, vs) # return (inp, loc, trace) # tasks = [(inp, l) for inp, lines in raw_traces.items() for l in lines] # wrs = Miscs.run_mp_ex("merge traces", tasks, f) # itraces = defaultdict(dict) # for inp, loc, trace in wrs: # itraces[inp].setdefault(loc, []).append(trace) # return itraces def f(task): inp, lines = task dtraces = defaultdict(list) for l in lines: # vtrace1: 8460 16 0 1 16 8460 parts = l.split(':') assert len(parts) == 2, parts loc, tracevals = parts[0], parts[1] loc = loc.strip() # vtrace1 ss = inv_decls[loc].names vs = tracevals.strip().split() trace = Trace.parse(ss, vs) dtraces[loc].append(trace) return (inp, dtraces) tasks = raw_traces.items() wrs = Miscs.run_mp_ex("merge traces", tasks, f) itraces = {inp: dtraces for inp, dtraces in wrs} return itraces
def myeval(self, expr): assert Miscs.is_expr(expr), expr rs = expr.subs(self.mydict) return rs
def symbols(self): try: return self._symbols except AttributeError: self._symbols = set(map(str, Miscs.get_vars(self.a + self.b))) return self._symbols
def symbols(self): try: return self._symbols except AttributeError: self._symbols = Miscs.get_vars(self.poly) return self._symbols
def __init__(self, poly): assert Miscs.is_expr(poly), poly super().__init__(poly)
def mysorted(ps): return sorted(ps, key=lambda p: len(Miscs.get_vars(p.inv)))
def get_auto_deg(self, maxdeg): maxvars = max(self.inv_decls.values(), key=lambda d: len(d)) deg = Miscs.get_auto_deg(maxdeg, len(maxvars), settings.MAX_TERM) return deg