def checkReach(self):
#check for reachability using inv False (0)
dinvs = DInvs.mkFalses(self.invdecls)
inps = Inps()
#use some initial inps first
rinps = Miscs.genInitInps(len(self.inpdecls), DTraces.inpMaxV)
logger.debug("gen {} random inps".format(len(rinps)))
for inp in rinps:
inps.add(Inp(inp))
traces = self.getTraces(inps)
unreachLocs = [loc for loc in dinvs if loc not in traces]
if unreachLocs:
logger.debug("use RT to generate traces for {}".format(','.join(
map(str, unreachLocs))))
unreachInvs = DInvs.mkFalses(unreachLocs)
cInps, _, _ = self.prover.checkRange(unreachInvs, inps=None)
newInps = self.updateInps(cInps, inps)
_ = self.getTracesAndUpdate(newInps, traces)
#remove FALSE invs indicating unreached locs
for loc in traces:
assert traces[loc]
dinvs[loc].clear()
return dinvs, traces, inps
def getInpsSafe(self, dinvs, inps, inpsd):
"""call verifier on each inv"""
def wprocess(tasks, Q):
rs = [(loc, inv,
self.src.instrAsserts(
{loc:set([inv])}, inps, inpsd,self.invdecls))
for loc, inv in tasks]
rs = [(loc, inv, KLEE(isrc, self.tmpdir).getDInps())
for loc, inv, isrc in rs]
if Q is None: #no multiprocessing
return rs
else:
Q.put(rs)
tasks = [(loc, inv) for loc in dinvs for inv in dinvs[loc]
if inv.stat is None]
wrs = Miscs.runMP("prove", tasks, wprocess, chunksiz=1,
doMP=settings.doMP and len(tasks) >= 2)
mInps, mCexs, mdinvs = [], [], DInvs()
for loc, inv, (klDInps, klDCexs, isSucc) in wrs:
mInps.append(klDInps)
mCexs.append(klDCexs)
try:
_ = klDInps[loc][str(inv)]
stat = Inv.DISPROVED
except KeyError:
stat = Inv.PROVED if isSucc else Inv.UNKNOWN
inv.stat = stat
if loc not in mdinvs: mdinvs[loc] = Invs()
mdinvs[loc].add(inv)
return merge(mInps), merge(mCexs), mdinvs
def gen(self, deg, traces, inps):
assert deg >= 1, deg
assert isinstance(traces, DTraces) and traces, traces
assert isinstance(inps, Inps), inps
#first obtain enough traces
initrs = [self.getInitTraces(loc, deg, traces, inps) for loc in traces]
tasks = [(loc, rs) for loc, rs in zip(traces, initrs) if rs]
#then solve/prove in parallel
def wprocess(tasks, Q):
rs = [(loc, self.infer(loc, template, uks, exprs, traces, inps))
for loc, (template, uks, exprs) in tasks]
if Q is None:
return rs
else:
Q.put(rs)
wrs = Miscs.runMP('find eqts',
tasks,
wprocess,
chunksiz=1,
doMP=settings.doMP and len(tasks) >= 2)
dinvs = DInvs()
for loc, (eqts, newInps) in wrs:
newInps = Gen.updateInps(newInps, inps)
logger.debug("{}: got {} eqts, {} new inps".format(
loc, len(eqts), len(newInps)))
if eqts: logger.debug('\n'.join(map(str, eqts)))
dinvs[loc] = Invs.mk(eqts)
return dinvs
def getInitTraces(self, loc, deg, traces, inps, rate=1.7):
vs = tuple(self.invdecls[loc])
terms = Miscs.getTerms([sage.all.var(k) for k in vs], deg)
template, uks = Miscs.mkTemplate(terms, 0, retCoefVars=True)
nEqtsNeeded = int(rate * len(uks))
exprs = traces[loc].instantiate(template, nEqtsNeeded)
while nEqtsNeeded > len(exprs):
logger.debug("{}: need more traces ({} eqts, need >= {})".format(
loc, len(exprs), nEqtsNeeded))
dinvsFalse = DInvs.mkFalses([loc])
cInps, _, _ = self.prover.checkRange(dinvsFalse, inps)
if loc not in cInps:
logger.warn("{}: cannot generate enough traces".format(loc))
return
newInps = Gen.updateInps(cInps, inps)
newTraces = self.getTracesAndUpdate(newInps, traces)
if loc not in newTraces:
logger.warn("I got a bad start and am stuck. Just restart me")
exit(0)
logger.debug("obtain {} new traces".format(len(newTraces[loc])))
newExprs = newTraces[loc].instantiate(template,
nEqtsNeeded - len(exprs))
for expr in newExprs:
assert expr not in exprs
exprs.add(expr)
return template, uks, exprs
def guessCheck(self, loc, term, minV, maxV, ubMinV, ubMaxV, disproves):
assert minV <= maxV, (minV, maxV, term)
assert ubMinV < ubMaxV, (ubMinV, ubMaxV)
#assert isinstance(traces, DTraces), traces
assert isinstance(disproves, set), disproves
if minV == maxV: return maxV
elif maxV - minV == 1:
if minV in disproves:
return maxV
inv = Inv(term <= minV)
inv_ = DInvs.mk(loc, Invs.mk([inv]))
_, dCexs, _ = self.prover.check(inv_, None, ubMinV, ubMaxV)
if loc in dCexs:
assert dCexs[loc]
disproves.add(minV)
return maxV
else:
return minV
v = sage.all.ceil((maxV + minV) / 2.0)
inv = Inv(term <= v)
inv_ = DInvs.mk(loc, Invs.mk([inv]))
_, dCexs, _ = self.prover.check(inv_, None, ubMinV, ubMaxV)
if loc in dCexs: #disproved
assert dCexs[loc]
cexs = Traces.extract(dCexs[loc],
tuple(self.invdecls[loc]),
useOne=False)
minV = int(max(cexs.myeval(term)))
disproves.add(v)
else:
maxV = v
return self.guessCheck(
loc,
term, #traces, inps,
minV,
maxV,
ubMinV,
ubMaxV,
disproves)
def infer(self, loc, template, uks, exprs, dtraces, inps):
assert isinstance(loc, str), loc
assert sageutil.is_sage_expr(template), template
assert isinstance(uks, list), uks
assert isinstance(exprs, set) and exprs, exprs
assert isinstance(dtraces, DTraces) and dtraces, dtraces
assert isinstance(inps, Inps) and inps, inps
vs = tuple(self.invdecls[loc])
cache = set()
eqts = set() #results
exprs = list(exprs)
newInps = []
curIter = 0
while True:
curIter += 1
logger.debug("{}: iter {} infer using {} exprs".format(
loc, curIter, len(exprs)))
newEqts = Miscs.solveEqts(exprs, uks, template)
unchecks = [eqt for eqt in newEqts if eqt not in cache]
if not unchecks:
logger.debug("{}: no new results -- break".format(loc))
break
logger.debug('{}: {} candidates:\n{}'.format(
loc, len(newEqts), '\n'.join(map(str, newEqts))))
logger.debug("{}: check {} unchecked ({} candidates)".format(
loc, len(unchecks), len(newEqts)))
dinvs = DInvs.mk(loc, Invs.mk(map(Inv, unchecks)))
dInps, dCexs, dinvs = self.prover.checkRange(dinvs, inps=None)
if dInps: newInps.append(dInps)
_ = [eqts.add(inv) for inv in dinvs[loc] if not inv.isDisproved]
_ = [
cache.add(inv.inv) for inv in dinvs[loc]
if inv.stat is not None
]
if loc not in dCexs:
logger.debug(
"{}: no disproved candidates -- break".format(loc))
break
cexs = Traces.extract(dCexs[loc], vs)
exprs_ = cexs.instantiate(template, None)
logger.debug("{}: {} new cex exprs".format(loc, len(exprs_)))
exprs.extend(exprs_)
return eqts, newInps
def naive(self, loc, term, minV, maxV, ubMinV, ubMaxV, disproves):
assert minV <= maxV, (minV, maxV, term)
assert ubMinV < ubMaxV, (ubMinV, ubMaxV)
#assert isinstance(traces, DTraces), traces
assert isinstance(disproves, set), disproves
for v in range(minV - 1, maxV + 1):
inv = Inv(term <= v)
inv_ = DInvs.mk(loc, Invs.mk([inv]))
_, dCexs, _ = self.prover.check(inv_, None, ubMinV, ubMaxV)
if loc in dCexs: #disproved
assert dCexs[loc]
cexs = Traces.extract(dCexs[loc],
tuple(self.invdecls[loc]),
useOne=False)
minV = int(max(cexs.myeval(term)))
disproves.add(v)
else:
return v
def start(self, seed, maxdeg, maxterm, doEqts, doIeqs):
assert isinstance(seed, (int, float)), seed
assert isinstance(doEqts, bool), doEqts
assert isinstance(doIeqs, bool), doIeqs
from time import time
st = time()
import random
random.seed(seed)
sage.all.set_random_seed(seed)
logger.info('set seed to: {} (test {})'.format(
seed, sage.all.randint(0, 100)))
##determine degree
maxvars = max(self.invdecls.itervalues(), key=lambda d: len(d))
deg = Miscs.getAutoDeg(maxdeg, maxterm, len(maxvars))
solver = Gen(self.inpdecls, self.invdecls, self.tcsFile, self.exeFile,
self.prover)
logger.info("check reachability")
dinvs, traces, inps = solver.checkReach()
if not traces:
return dinvs
def strOfLocs(locs):
_f = lambda vts: ', '.join("{} {}".format(vts[v], v) for v in vts)
s = '; '.join("{} ({})".format(loc, _f(self.invdecls[loc]))
for loc in locs)
return "{} locs: {}".format(len(locs), s)
def _gen(typ):
st_gen = time()
cls = GenEqts if typ == 'eqts' else GenIeqs
logger.info("gen {} at {}".format(typ, strOfLocs(traces.keys())))
solver = cls(self.inpdecls, self.invdecls, self.tcsFile,
self.exeFile, self.prover)
invs = solver.gen(deg, traces, inps)
logger.info("gen {}: ({}s)".format(typ, time() - st_gen))
if invs:
dinvs.merge(invs)
logger.info("{} invs:\n{}".format(dinvs.siz, dinvs))
if doEqts: _gen('eqts')
if doIeqs: _gen('ieqs')
logger.info("test {} invs on all {} traces".format(
dinvs.siz, traces.siz))
dinvs = dinvs.testTraces(traces)
logger.info("find uniq invs")
st_uniq = time()
logger.info("{} invs:\n{}".format(dinvs.siz, dinvs))
oldSiz = dinvs.siz
def wprocess(tasks, Q):
rs = [(loc, Miscs.reduceSMT(invs)) for loc, invs in tasks]
if Q is None:
return rs
else:
Q.put(rs)
tasks = [(loc, [inv.inv for inv in dinvs[loc]]) for loc in dinvs]
wrs = Miscs.runMP("uniqify",
tasks,
wprocess,
chunksiz=1,
doMP=settings.doMP and len(tasks) >= 2)
dinvs = DInvs((loc, Invs(map(Inv, invs))) for loc, invs in wrs if invs)
logger.debug("uniqify: remove {} redundant invs ({}s)".format(
oldSiz - dinvs.siz,
time() - st_uniq))
logger.info(
"*** {}, {} locs, invs {}, inps {}, time {} s, rand {}: \n{}".
format(self.filename, len(dinvs), dinvs.siz, len(inps),
time() - st, sage.all.randint(0, 100), dinvs))
import shutil
logger.debug("rm -rf {}".format(self.tmpdir))
shutil.rmtree(self.tmpdir)
return dinvs
def gen(self, deg, traces, inps):
assert deg >= 1, deg
assert isinstance(traces, DTraces), traces
assert isinstance(inps, Inps), inps
assert isinstance(traces, DTraces) and traces, traces
assert isinstance(inps, Inps), inps
mymaxv = 10
maxV = mymaxv
minV = -1 * maxV
#without these restrictions, klee takes a long time to run
ubmaxV = maxV * 2
ubminV = -1 * ubmaxV
locs = traces.keys()
vss = [[sage.all.var(k) for k in self.invdecls[loc]] for loc in locs]
mydeg = 2
if mydeg > 2:
logger.warn("not Oct invs (deg {}). Might be slow".format(deg))
termss = [Miscs.getTermsFixedCoefs(vs, mydeg) for vs in vss]
logger.info(
"{} locs: check upperbounds for {} terms (range {})".format(
len(locs), sum(map(len, termss)), mymaxv))
refs = {
loc: {Inv(t <= maxV): t
for t in terms}
for loc, terms in zip(locs, termss)
}
ieqs = DInvs((loc, Invs.mk(refs[loc].keys())) for loc in refs)
myinps = None
cInps, cTraces, ieqs = self.prover.check(ieqs, myinps, ubminV, ubmaxV)
if cInps:
newInps = Gen.updateInps(cInps, inps)
_ = self.getTracesAndUpdate(newInps, traces)
ieqs = ieqs.removeDisproved()
tasks = [(loc, refs[loc][ieq]) for loc in ieqs for ieq in ieqs[loc]]
logger.debug("{} locs: compute upperbounds for {} terms".format(
len(locs), len(tasks)))
def _f(loc, term):
vs = traces[loc].myeval(term)
try:
mminV = int(max(minV, max(v for v in vs if v < maxV)))
except ValueError:
mminV = minV
logger.debug(
"{}: compute ub for '{}', start w/ min {}, maxV {})".format(
loc, term, mminV, maxV))
disproves = set()
boundV = self.guessCheck(
loc,
term, #traces, inps,
mminV,
maxV,
ubminV,
ubmaxV,
disproves)
if boundV not in disproves and boundV not in {maxV, minV}:
inv = Inv(term <= boundV)
logger.detail("got {}".format(inv))
return inv
else:
return None
def wprocess(tasks, Q):
rs = [(loc, _f(loc, term)) for loc, term in tasks]
if Q is None:
return rs
else:
Q.put(rs)
doMP = settings.doMP and len(tasks) >= 2
wrs = Miscs.runMP('guesscheck', tasks, wprocess, chunksiz=1, doMP=doMP)
rs = [(loc, inv) for loc, inv in wrs if inv]
dinvs = DInvs()
for loc, inv in rs:
if loc not in dinvs: dinvs[loc] = Invs()
dinvs[loc].add(inv)
return dinvs