def _simplify(cls, invs, is_conj, use_reals):
assert invs, invs
st = time()
eqts, eqts_largecoefs, octs, mps, preposts, falseinvs = \
cls.classify(invs)
def mysorted(ps):
return sorted(ps, key=lambda p: len(Miscs.get_vars(p.inv)))
eqts = mysorted(eqts + eqts_largecoefs)
octs = mysorted(octs)
mps = mysorted(mps)
myinvs = eqts + falseinvs + preposts + octs + mps
myinvs_exprs = [inv.expr(use_reals) for inv in myinvs]
def _imply(js, i):
iexpr = myinvs_exprs[i]
# don't consider/remove equality
if iexpr.decl().kind() == z3.Z3_OP_EQ:
ret = False
else:
jexprs = [myinvs_exprs[j] for j in js]
ret = Z3._imply(jexprs, iexpr, is_conj)
# if ret:
# print '{} => {}'.format(jexprs, iexpr)
return ret
results = Miscs.simplify_idxs(list(range(len(myinvs))), _imply)
results = [myinvs[i] for i in results]
Miscs.show_removed('_simplify', len(invs), len(results), time() - st)
return results
def filter_terms(terms, inps):
assert isinstance(inps, set) and \
all(isinstance(s, str) for s in inps), inps
if not inps:
mlog.warning("Have not tested case with no inps")
excludes = set()
for term in terms:
if isinstance(term, data.poly.mp.MP):
a_symbs = list(map(str, Miscs.get_vars(term.a)))
b_symbs = list(map(str, Miscs.get_vars(term.b)))
inp_in_a = any(s in inps for s in a_symbs)
inp_in_b = any(s in inps for s in b_symbs)
if ((inp_in_a and inp_in_b)
or (not inp_in_a and not inp_in_b)):
excludes.add(term)
continue
t_symbs = set(a_symbs + b_symbs)
else:
t_symbs = set(map(str, term.symbols))
if len(t_symbs) <= 1: # ok for finding bound of single input val
continue
if (inps.issuperset(t_symbs)
or all(s not in inps for s in t_symbs)):
excludes.add(term)
new_terms = [term for term in terms if term not in excludes]
return new_terms
def _get_init_traces(self, loc, deg, traces, inps, rate):
"Initial loop to obtain (random) traces to bootstrap eqt solving"
assert deg >= 1, deg
assert isinstance(rate, float) and rate >= 0.1, rate
if self.use_rand_init:
whileF, whileFName = self._while_rand, "random"
else:
whileF, whileFName = self._while_symstates, "symstates"
mlog.debug(
"{}: gen init inps using {} (curr inps {}, traces {})".format(
loc, whileFName, len(inps), len(traces)))
terms, template, uks, n_eqts_needed = Miscs.init_terms(
self.inv_decls[loc].names, deg, rate)
exprs = whileF(loc, template, n_eqts_needed, inps, traces)
# if cannot generate sufficient traces, adjust degree
while (not exprs):
if deg < 2:
return # cannot generate enough traces
deg = deg - 1
mlog.info(
"Reduce polynomial degree to {}, terms {}, uks {}".format(
deg, len(terms), len(uks)))
terms, template, uks, n_eqts_needed = Miscs.init_terms(
self.inv_decls[loc].names, deg, rate)
exprs = whileF(loc, template, n_eqts_needed, inps, traces)
return template, uks, exprs
def infer_eqts(self, maxdeg, symbols, traces):
auto_deg = self.get_auto_deg(maxdeg)
terms, template, uks, n_eqts_needed = Miscs.init_terms(
symbols.names, auto_deg, settings.EQT_RATE)
exprs = list(traces.instantiate(template, n_eqts_needed))
eqts = Miscs.solve_eqts(exprs, uks, template)
import data.inv.eqt
return [data.inv.eqt.Eqt(eqt) for eqt in eqts]
def _infer(self, loc, template, uks, exprs, dtraces, inps):
assert isinstance(loc, str) and loc, loc
assert Miscs.is_expr(template), template
assert isinstance(uks, list), uks
assert isinstance(exprs, set) and exprs, exprs
assert isinstance(dtraces, DTraces) and dtraces, dtraces
assert isinstance(inps, Inps) and inps, inps
cache = set()
eqts = set() # results
exprs = list(exprs)
new_cexs = []
curIter = 0
while True:
curIter += 1
mlog.debug("{}, iter {} infer using {} exprs".format(
loc, curIter, len(exprs)))
new_eqts = Miscs.solve_eqts(exprs, uks, template)
unchecks = [eqt for eqt in new_eqts if eqt not in cache]
if not unchecks:
mlog.debug("{}: no new results -- break".format(loc))
break
mlog.debug('{}: {} candidates:\n{}'.format(
loc, len(new_eqts), '\n'.join(map(str, new_eqts))))
mlog.debug("{}: check {} unchecked ({} candidates)".format(
loc, len(unchecks), len(new_eqts)))
dinvs = DInvs.mk(loc, Invs(list(map(data.inv.eqt.Eqt, unchecks))))
cexs, dinvs = self.check(dinvs, None)
if cexs:
new_cexs.append(cexs)
[eqts.add(inv) for inv in dinvs[loc] if not inv.is_disproved]
[cache.add(inv.inv) for inv in dinvs[loc] if inv.stat is not None]
if loc not in cexs:
mlog.debug("{}: no disproved candidates -- break".format(loc))
break
cexs = Traces.extract(cexs[loc])
cexs = cexs.padzeros(set(self.inv_decls[loc].names))
exprs_ = cexs.instantiate(template, None)
mlog.debug("{}: {} new cex exprs".format(loc, len(exprs_)))
exprs.extend(exprs_)
return eqts, new_cexs
def test(self, dtraces):
# assert isinstance(dtraces, DTraces)
assert self.siz, self
st = time()
tasks = [loc for loc in self if self[loc]]
def f(tasks):
return [(loc, self[loc].test(dtraces[loc])) for loc in tasks]
wrs = Miscs.run_mp("test_dinvs", tasks, f)
dinvs = DInvs([(loc, invs) for loc, invs in wrs if invs])
Miscs.show_removed("test_dinvs", self.siz, dinvs.siz, time() - st)
return dinvs
def simplify(self, use_reals):
assert isinstance(use_reals, bool), use_reals
assert (self.siz), self
st = time()
def f(tasks):
return [(loc, self[loc].simplify(use_reals)) for loc in tasks]
wrs = Miscs.run_mp('simplify', list(self), f)
dinvs = self.__class__((loc, invs) for loc, invs in wrs if invs)
Miscs.show_removed('simplify', self.siz, dinvs.siz, time() - st)
return dinvs
def gen(self, deg, traces, inps):
assert deg >= 1, deg
assert isinstance(traces, DTraces) and traces, traces
assert isinstance(inps, Inps), inps
locs = traces.keys()
# first obtain enough traces
tasks = [(loc,
self._get_init_traces(loc, deg, traces, inps,
settings.EQT_RATE)) for loc in locs]
tasks = [(loc, tcs) for loc, tcs in tasks if tcs]
# then solve/prove in parallel
def f(tasks):
return [(loc, self._infer(loc, template, uks, exprs, traces, inps))
for loc, (template, uks, exprs) in tasks]
wrs = Miscs.run_mp('find eqts', tasks, f)
# put results together
dinvs = DInvs()
for loc, (eqts, cexs) in wrs:
new_inps = inps.merge(cexs, self.inp_decls.names)
mlog.debug("{}: got {} eqts, {} new inps".format(
loc, len(eqts), len(new_inps)))
if eqts:
mlog.debug('\n'.join(map(str, eqts)))
dinvs[loc] = Invs(eqts)
return dinvs
def myeval(self, expr, pred=None):
assert Miscs.is_expr(expr), expr
if pred is None:
return [trace.myeval(expr) for trace in self]
else:
return any(pred(trace.myeval(expr)) for trace in self)
def get_preconds(cls, symbols, term_siz):
"""
sage: x,y,z = sage.all.var('x y z')
#doctest: +NORMALIZE_WHITESPACE
sage: sorted(CegirPrePosts._preconds([x,y], 2), key=str)
[-x + y < 0,
-x + y <= 0,
-x - y < 0,
-x - y <= 0,
-x < 0,
-x <= 0,
-y < 0,
-y <= 0,
x + y < 0,
x + y <= 0,
x - y < 0,
x - y <= 0,
x < 0,
x <= 0,
x == 0,
y < 0,
y <= 0,
y == 0]
"""
t1 = [Eqt(t == 0) for t in symbols] # M=0, N=0
ts = Miscs.get_terms_fixed_coefs(symbols, term_siz, settings.ICOEFS)
t2 = [Oct(t < 0) for t in ts] # +/M+/-N >0
t3 = [Oct(t <= 0) for t in ts] # +/M+/-N >=0
return t1 + t2 + t3
def simplify(self, use_reals):
assert isinstance(use_reals, bool), use_reals
eqts, eqts_largecoefs, octs, mps, preposts, falseinvs = \
self.classify(self)
assert not falseinvs, falseinvs
non_mps = eqts + preposts + octs
if non_mps and len(mps) >= 2: # parallelizing simplifying mps
non_mps_exprs = [e.expr(use_reals) for e in non_mps]
conj = z3.And(non_mps_exprs)
def f(mps):
return [
mp for mp in mps if not Z3._imply(conj, mp.expr(use_reals))
]
wrs = Miscs.run_mp("simplifying {} mps".format(len(mps)), mps, f)
mps = [mp for mp in wrs]
rs = non_mps + mps
if rs:
is_conj = True
rs = self._simplify(rs, is_conj, use_reals)
return self.__class__(rs + eqts_largecoefs)
def prove_reach(self, vs, input):
# @timeit
def f(task):
vid, vld_cls = task
vld = vld_cls(self.tmpdir)
r, cex = vld.prove_reach(vs, input)
if r is None:
return None
else:
return vid, (r, cex)
wrs = Miscs.run_mp_ex(
"prove_reach",
[(settings.CPAchecker.CPA_SHORT_NAME, CPAchecker),
(settings.Ultimate.UAUTOMIZER_SHORT_NAME, UAutomizer),
(settings.Ultimate.UTAIPAN_SHORT_NAME, UTaipan)],
f,
get_fst_res=True)
mlog.debug('wrs: {}'.format(wrs))
if wrs:
vid, r = wrs[0]
mlog.debug('Got result firstly from {}'.format(vid))
return r
else:
return None, None
def get_postconds(cls, eqt):
assert Miscs.is_expr(eqt), eqt
assert eqt.operator() == operator.eq, eqt
# tCtr
symbols = [
s for s in Miscs.get_vars(eqt) if settings.CTR_VAR in str(s)
]
if not symbols:
return
assert len(symbols) == 1, \
"should only have 1 tCtr symbol: {}, {}".format(
symbols, settings.CTR_VAR)
postconds = sage.all.solve(eqt, symbols[0])
return postconds if len(postconds) >= 1 else None
def test_single_trace(self, trace):
assert isinstance(trace, data.traces.Trace), trace
# temp fix: disable repeating rational when testing equality
if (any(not x.is_integer() and Miscs.is_repeating_rational(x)
for x in trace.vs)):
mlog.debug("skip trace with repeating rational {}".format(self))
return True
return super().test_single_trace(trace)
def get_terms(self, symbols):
terms = []
if settings.DO_IEQS:
oct_siz = 2
terms_ieqs = Miscs.get_terms_fixed_coefs(symbols, oct_siz)
terms_ieqs = [data.poly.base.GeneralPoly(t) for t in terms_ieqs]
mlog.debug("{} terms for Ieqs".format(len(terms_ieqs)))
terms.extend(terms_ieqs)
if settings.DO_MINMAXPLUS:
terms_u = data.poly.mp.MP.get_terms(symbols)
terms_u_no_octs = [(a, b) for a, b in terms_u if len(b) >= 2]
if settings.DO_IEQS: # ignore oct invs
terms_u = terms_u_no_octs
def _get_terms(terms_u, is_max):
terms_l = [(b, a) for a, b in terms_u]
terms = terms_u + terms_l
terms = [data.poly.mp.MP(a, b, is_max) for a, b in terms]
return terms
terms_max = _get_terms(terms_u, is_max=True)
terms_min = _get_terms(terms_u_no_octs, is_max=False)
terms_mp = terms_min + terms_max
terms.extend(terms_mp)
mlog.debug("{} terms for MP".format(len(terms_mp)))
print(terms_mp)
if settings.DO_TERM_FILTER:
st = time()
new_terms = self.filter_terms(terms,
set(self.prog.inp_decls.names))
Miscs.show_removed('term filter', len(terms), len(new_terms),
time() - st)
return new_terms
else:
return terms
def instantiate(self, term, ntraces):
assert Miscs.is_expr(term), term
assert ntraces is None or ntraces >= 1, ntraces
if ntraces is None:
for t in self.mydicts:
exprs = set(term.subs(t) for t in self.mydicts)
else:
ntracesExtra = ntraces * settings.TRACE_MULTIPLIER
exprs = set()
for t in self.mydicts:
expr = term.subs(t)
if expr not in exprs:
exprs.add(expr)
if len(exprs) >= ntracesExtra:
break
# instead of doing this, can find out the # 0's in traces
# the more 0's , the better
exprs = sorted(exprs, key=lambda expr: len(Miscs.get_vars(expr)))
exprs = set(exprs[:ntraces])
return exprs
def get_conj_preconds(self, loc, preconds, postcond_expr):
"""
preconds => post can be strengthened by removing some preconds
e.g., a&b => post is stronger than a&b&c => post
"""
assert all(isinstance(p, Inv) for p in preconds), preconds
assert z3.is_expr(postcond_expr), postcond_expr
if not preconds:
return []
preconds = sorted(preconds, key=lambda p: len(Miscs.get_vars(p.inv)))
preconds_exprs = [pc.expr(self.use_reals) for pc in preconds]
if not self.check(preconds_exprs, postcond_expr, loc):
return []
def _imply(js, _):
jexprs = [preconds_exprs[j] for j in js]
return self.check(jexprs, postcond_expr, loc)
results = Miscs.simplify_idxs(list(range(len(preconds))), _imply)
results = [preconds[i] for i in results]
return results
def gen(self, traces, inps):
assert isinstance(traces, data.traces.DTraces) and traces, traces
assert isinstance(inps, data.traces.Inps), inps
statss = []
locs = traces.keys()
termss = [
self.get_terms(self.inv_decls[loc].sageExprs) for loc in locs
]
mlog.debug("check upperbounds for {} terms at {} locs".format(
sum(map(len, termss)), len(locs)))
maxV = settings.OCT_MAX_V
minV = -1 * maxV
refs = {
loc: {self.mk_le(t, maxV): t
for t in terms}
for loc, terms in zip(locs, termss)
}
ieqs = data.inv.invs.DInvs([(loc, data.inv.invs.Invs(refs[loc].keys()))
for loc in refs])
cexs, ieqs, stats = self.check(ieqs, inps=None)
statss.extend(stats)
if cexs:
cexs_inps = inps.merge(cexs, self.inp_decls.names)
if cexs_inps:
self.get_traces(cexs_inps, traces)
ieqs = ieqs.remove_disproved()
tasks = [(loc, refs[loc][mp]) for loc in ieqs for mp in ieqs[loc]]
mlog.debug("{} locs: infer upperbounds for {} terms".format(
len(locs), len(tasks)))
def f(tasks):
return [(loc, self.gc(loc, term, minV, maxV, traces))
for loc, term in tasks]
wrs = Miscs.run_mp('guesscheck', tasks, f)
dinvs = data.inv.invs.DInvs()
for loc, (inv, stats) in wrs:
statss.extend(stats)
if inv:
dinvs.setdefault(loc, data.inv.invs.Invs()).add(inv)
return dinvs, statss
def start(self, seed, maxdeg):
assert maxdeg is None or maxdeg >= 1, maxdeg
super().start(seed, maxdeg)
st = time.time()
tasks = []
for loc in self.dtraces:
symbols = self.inv_decls[loc]
traces = self.dtraces[loc]
if settings.DO_EQTS:
def _f():
return self.infer_eqts(maxdeg, symbols, traces)
tasks.append((loc, _f))
if settings.DO_IEQS:
def _f():
return self.infer_ieqs(symbols, traces)
tasks.append((loc, _f))
def f(tasks):
rs = [(loc, _f()) for loc, _f in tasks]
return rs
wrs = Miscs.run_mp("(pure) dynamic inference", tasks, f)
dinvs = DInvs()
for loc, invs in wrs:
for inv in invs:
dinvs.add(loc, inv)
try:
new_traces = self.dtraces.merge(self.test_dtraces)
mlog.debug('added {} test traces'.format(new_traces.siz))
except AttributeError:
# no test traces
pass
dinvs = self.sanitize(dinvs, self.dtraces)
print(dinvs)
def test(self, traces):
# assert isinstance(traces, Traces)
assert (self), self
def f(tasks):
return [(inv, inv.test(traces)) for inv in tasks]
wrs = Miscs.run_mp("test", list(self), f)
myinvs = set()
for inv, passed in wrs:
if passed:
myinvs.add(inv)
else:
mlog.debug("remove {}".format(inv))
invs = self.__class__(myinvs)
return invs
def _get_traces_mp(self, inps):
"""
run program on inps and obtain traces in parallel
return {inp: traces}
"""
assert isinstance(inps, data.traces.Inps), inps
tasks = [inp for inp in inps if inp not in self._cache]
def f(tasks):
return [(inp, self._get_traces(inp)) for inp in tasks]
wrs = Miscs.run_mp("get traces", tasks, f)
for inp, traces in wrs:
assert inp not in self._cache
self._cache[inp] = traces
return {inp: self._cache[inp] for inp in inps}
def classify(cls, invs):
eqts, eqts_largecoefs, octs, mps, preposts, falseinvs = [], [], [], [], [], []
for inv in invs:
if isinstance(inv, data.inv.eqt.Eqt):
if len(Miscs.get_coefs(inv.inv)) > 10:
eqts_largecoefs.append(inv)
else:
eqts.append(inv)
elif isinstance(inv, data.inv.oct.Oct):
octs.append(inv)
elif isinstance(inv, data.inv.mp.MP):
mps.append(inv)
elif isinstance(inv, data.inv.prepost.PrePost):
preposts.append(inv)
else:
assert isinstance(inv, data.inv.invs.FalseInv), inv
falseinvs.append(inv)
return eqts, eqts_largecoefs, octs, mps, preposts, falseinvs
def infer_ieqs(self, symbols, traces):
maxV = settings.IUPPER
minV = -1 * maxV
terms = Miscs.get_terms_fixed_coefs(
symbols.sageExprs,
settings.ITERMS,
settings.ICOEFS,
)
ieqs = []
for t in terms:
upperbound = max(traces.myeval(t))
if upperbound > maxV or upperbound < minV:
continue
ieqs.append(t <= upperbound)
import data.inv.oct
ieqs = [data.inv.oct.Oct(ieq) for ieq in ieqs]
return ieqs
def _merge_traces():
# def f(task):
# inp, l = task
# # vtrace1: 8460 16 0 1 16 8460
# parts = l.split(':')
# assert len(parts) == 2, parts
# loc, tracevals = parts[0], parts[1]
# loc = loc.strip() # vtrace1
# ss = inv_decls[loc].names
# vs = tracevals.strip().split()
# trace = Trace.parse(ss, vs)
# return (inp, loc, trace)
# tasks = [(inp, l) for inp, lines in raw_traces.items() for l in lines]
# wrs = Miscs.run_mp_ex("merge traces", tasks, f)
# itraces = defaultdict(dict)
# for inp, loc, trace in wrs:
# itraces[inp].setdefault(loc, []).append(trace)
# return itraces
def f(task):
inp, lines = task
dtraces = defaultdict(list)
for l in lines:
# vtrace1: 8460 16 0 1 16 8460
parts = l.split(':')
assert len(parts) == 2, parts
loc, tracevals = parts[0], parts[1]
loc = loc.strip() # vtrace1
ss = inv_decls[loc].names
vs = tracevals.strip().split()
trace = Trace.parse(ss, vs)
dtraces[loc].append(trace)
return (inp, dtraces)
tasks = raw_traces.items()
wrs = Miscs.run_mp_ex("merge traces", tasks, f)
itraces = {inp: dtraces for inp, dtraces in wrs}
return itraces
def myeval(self, expr):
assert Miscs.is_expr(expr), expr
rs = expr.subs(self.mydict)
return rs
def symbols(self):
try:
return self._symbols
except AttributeError:
self._symbols = set(map(str, Miscs.get_vars(self.a + self.b)))
return self._symbols
def symbols(self):
try:
return self._symbols
except AttributeError:
self._symbols = Miscs.get_vars(self.poly)
return self._symbols
def __init__(self, poly):
assert Miscs.is_expr(poly), poly
super().__init__(poly)
def mysorted(ps):
return sorted(ps, key=lambda p: len(Miscs.get_vars(p.inv)))
def get_auto_deg(self, maxdeg):
maxvars = max(self.inv_decls.values(), key=lambda d: len(d))
deg = Miscs.get_auto_deg(maxdeg, len(maxvars), settings.MAX_TERM)
return deg
