def verify(lvl,
invs,
timeout=None,
overfittedSet=None,
nonindSet=None,
soundSet=None):
(overfitted,
_), (nonind,
_), sound, violations = tryAndVerifyLvl(lvl, invs, set(), timeout)
print("OVERFITTED:", overfitted)
print("NONIND:", nonind)
print("SOUND:", sound)
print("VIOLATIONS:", violations)
print()
print("PROVED:", not violations)
print()
if overfittedSet is not None:
overfittedSet.update(x[0] for x in overfitted)
if nonindSet is not None:
nonindSet.update(x[0] for x in nonind)
if soundSet is not None:
soundSet.update(sound)
return not violations
def really_verified_by_play(lvl, assignment, worker, exp, play, timeout):
invs = set([
parseExprAst(x.payl()['canonical']) for x in play
if x.type == 'FoundInvariant'
])
(overfitted,
_), (nonind,
_), sound, violations = tryAndVerifyLvl(lvl, invs, set(), timeout)
assert type(violations) == list
return len(violations) == 0
invs = list(map(z3_expr_to_boogie, loopInvs))
except Exception as e:
if (e.message == "timeout"):
invs = None
conf_status = "timeout"
else:
for i in loopInvs:
error(to_smt2(i))
raise
finally:
#alarm(0)
t.cancel()
if (invs != None):
try:
(overfitted, nonind, sound, violations) =\
tryAndVerifyLvl(lvl, set(invs), set(), args.time_limit, addSPs=True)
error("Out of ", invs, "sound: ", sound)
if (len(violations) > 0):
error("Supposedly sound inv: ", invs)
error("Level ", lvlName, "false claimed to be sound!")
error("Raw output: ", rawOutput)
conf_status = False
else:
conf_status = True
except Exception as e:
conf_status = "verification error: " + e.value
conf[lvlName] = conf_status
# if (args.csv_table):
# print lvlName, ",", res[lvlName][0], ",", conf_status
for precc in lvls[z]["splitterPreds"] ])
for z in originalToSplitM[origName] ]
implied = reduce(lambda x, y: x.union(y), implied, set())
found = set([bast.parseExprAst(x[1]) for x in found])
nraw = len(found)
nimplied = len(implied)
lvl = lvls[originalToSplitM[origName][0]]
bbs = lvl["program"]
loop = lvl["loop"]
loop_header = loop.loop_paths[0][0]
sps = propagate_sp(bbs)[loop_header]
nsps = len(sps)
invs = found.union(sps).union(implied)
(overfitted, dummy1), (nonind, dummy2), sound, violations =\
tryAndVerifyLvl(lvl, invs, set(), args.timeout, \
useSplitters=False, addSPs=False, generalizeUserInvs=False)
assert (len(dummy2) == 0 and len(dummy1) == 0)
solved = len(violations) == 0
print(",".join(map(str, [
origName,\
len(originalToSplitM[origName]),\
solved,\
nraw,\
nimplied,\
nsps,\
len(sound),\
len(overfitted),\
len(nonind)\
])))
help='Lvl-id in level set to try and verify"')
p.add_argument('invs', type=str, nargs="+", help="Invariants to try")
p.add_argument('--timeout', type=int, \
help="Optional z3 timeout", default=None)
args = p.parse_args();
_, lvls = loadBoogieLvlSet(args.lvlset)
lvl = lvls[args.lvlid]
boogie_invs = set([])
for inv in args.invs:
try:
bInv = parseExprAst(inv);
boogie_invs.add(bInv);
except Exception:
error("Failed parsing invariant " + inv);
exit(-1);
((overfitted, overfitted_p2), (nonind, nonind_p2), sound, violations) =\
tryAndVerifyLvl(lvl, boogie_invs, set(), args.timeout)
overfitted = set(overfitted).union(overfitted_p2)
nonind = set(nonind).union(nonind_p2)
print("Overfitted: ", overfitted)
print("Nonind: ", nonind)
print("Sound: ", sound)
print("Violations: ", violations)
print("Verified? ", len(violations) == 0)
t_header_vals = [ [ row[1][0][v] for v in vs ] \
for row in t_loop_headers ]
header_vals = header_vals + t_header_vals
invs = runDaikon(vs, header_vals, args.no_suppression)
binvs = [ ]
for inv in invs:
try:
binvs.append(daikonToBoogieExpr(inv))
except Exception:
if (not args.csv_table):
sys.stderr.write("Can't translate " + str(inv) + "\n");
((overfitted, overfitted_p2), (nonind, nonind_p2), soundInvs, violations) =\
tryAndVerifyLvl(lvl, binvs, set(), args.timeout, \
useSplitters = args.use_splitter_predicates)
overfitted += overfitted_p2
nonind += nonind_p2
if (args.check_solved):
solved = "," + str(len(violations) == 0)
else:
solved = "";
if (args.csv_table):
print(",".join([lvlName, \
str(len(binvs)), \
";".join(map(str, binvs)), str(len(soundInvs)), \
";".join(map(str, soundInvs)) ]) + solved)
else:
print("For lvl", lvlName, " daikon found", len(binvs), ":", binvs,\
def tryAndVerify(
levelSet: str,
levelId: str,
invs: List[EsprimaNode],
mturkId: MturkIdT,
individualMode: bool
) -> Tuple[List[Tuple[EsprimaNode, StoreArr]],\
List[Tuple[EsprimaNode, Tuple[StoreArr, StoreArr]]],\
List[EsprimaNode],\
List[StoreArr],
List[StoreArr]]:
"""
Given a level (levelSet, levelId) and a set of invaraints invs do:
1) Find all invariants OldI that were not DEFINITELY false from the
last time we tried to verify this level (getLastVerResult)
2) Try and verify the level using S = OldI + invs. This is done by
calling tryAndVerifyLvl. For more on that see comment in
tryAndVerifyLvl
3) Return object containing:
overfitted - all invariants in invs not implied by precondition
nonind - all invariants in invs that are not inductive
sound - all invariants in invs that are sound
post_ctrex - any safety counterexamples to the "sound" invariants.
If this set is empty, then the level is verified.
direct_ctrex - any safety counterexamples to ALL of the passed in
invs (not just the sound ones). This is used by the 'rounds' game
mode to generate red rows for the next level.
"""
s = sessionF()
if (levelSet not in traces):
raise Exception("Unkonwn level set " + str(levelSet))
if (levelId not in traces[levelSet]):
raise Exception("Unkonwn trace " + str(levelId) + \
" in levels " + str(levelSet))
if (len(invs) == 0):
raise Exception("No invariants given")
lvl: BoogieTraceLvl = traces[levelSet][levelId]
assert 'program' in lvl
workerId, _, _ = mturkId
userInvs = set([ esprimaToBoogie(x, {}) for x in invs ])
otherInvs: Set[AstExpr] = set([])
lastVer: Optional[Dict[str, Any]] = None
#lastVer = getLastVerResult(levelSet, levelId, s,
# workerId=(workerId if individualMode else None))
if (lastVer):
otherInvs = otherInvs.union([parseExprAst(x) for x in lastVer["sound"]])
otherInvs = otherInvs.union([parseExprAst(x) for x in lastVer["nonind"]])
((overfitted, _), (nonind, _), sound, violations) = \
tryAndVerifyLvl(lvl, userInvs, otherInvs, 1) # making all calls 1 sec args.timeout)
# See if the level is solved
solved = len(violations) == 0
fix = lambda x: _from_dict(lvl['variables'], x, 0)
# TODO: Why is this shit needed?
if (not solved):
fun: Function = lvl["program"]
direct_ctrexs = loopInvSafetyCtrex(fun, otherInvs.union(userInvs),\
1) # making all calls 1 sec args.timeout)
else:
direct_ctrexs = []
# Convert all invariants from Boogie to esprima expressions, and
# counterexamples to arrays
# from dictionaries
overfittedEsp = [ (boogieToEsprima(inv), fix(v.endEnv()))
for (inv, v) in overfitted ]
nonindEsp = [ (boogieToEsprima(inv), (fix(v.startEnv()), fix(v.endEnv())))
for (inv, v) in nonind ]
soundEsp = [ boogieToEsprima(inv) for inv in sound ]
safetyCtrexsArr = [ fix(v.startEnv()) for v in violations ]
directCtrexsArr = [ fix(v) for v in direct_ctrexs ]
res = (overfittedEsp, nonindEsp, soundEsp, safetyCtrexsArr, directCtrexsArr)
# Log verification result in Db (storing invariants as boogie strings)
payl = {
"lvlset": levelSet,
"lvlid": levelId,
"overfitted":nodups([str(inv) for (inv, _) in overfitted]),
"nonind":nodups([str(inv) for (inv,_) in nonind]),
"sound":nodups([str(inv) for inv in sound]),
"post_ctrex":safetyCtrexsArr,
"direct_ctrex": directCtrexsArr
}
addEvent("verifier", "VerifyAttempt", time(), args.ename, \
"localhost", payl, s, mturkId)
return res
def verify(aLvl, aInvs):
((aOverfit, aOverfit2), (aNonind, aNonind2), aSound, aViolations) =\
tryAndVerifyLvl(aLvl, set(aInvs), set(), args.timeout,
addSPs = False, useSplitters = False, generalizeUserInvs = False);
assert (len(aOverfit2) == 0 and len(aNonind2) == 0)
return (aOverfit, aNonind, aSound, aViolations)
error("Running ", lvlName)
mainRoutine = "main" if lvlName != "i-cegar1" else "cegar1"
res[lvlName] = runInvGen(invgenCppFile, mainRoutine)
solved, loopInvs, rawOutput = res[lvlName]
conf_status = "n/a"
if (solved == True):
error("z3 invs: ", len(loopInvs), loopInvs)
boogieInvs = list(map(parseInvGenInvariant, loopInvs))
bbs = lvl["program"]
loop = lvl["loop"]
try:
((overfitted, overfitted2), (nonind, nonind2), sound, violations) =\
tryAndVerifyLvl(lvl, set(boogieInvs), set(), args.time_limit, \
addSPs=True, useSplitters=False, generalizeUserInvs = False)
assert (len(overfitted2) == 0 and len(nonind2) == 0)
if (len(violations) > 0):
error("Supposedly sound inv: ", loopInvs)
error("Results : ", overfitted, nonind, sound)
error("Level ", lvlName, "false claimed to be sound!")
error("Raw output: ", rawOutput)
conf_status = False
else:
conf_status = True
except Exception as e:
if (e.message == "Unknown binary operator /"):
conf_status = "unhandled division"
else:
raise e
else:
