def loadBoogieLvlSet(lvlSetFile: str) -> Tuple[str, Dict[str, BoogieTraceLvl]]:
# Small helper funct to make sure we didn't
# accidentally give two levels the same name
def assertUniqueKeys(kvs):
keys = [x[0] for x in kvs]
assert (len(set(keys)) == len(keys))
return dict(kvs)
lvlSet = load(open(lvlSetFile, "r"), object_pairs_hook=assertUniqueKeys)
lvlSetDir = dirname(abspath(realpath(lvlSetFile)))
error("Loading level set " + lvlSet["name"] + " from " + lvlSetFile)
lvls: Dict[str, BoogieTraceLvl] = OrderedDict()
for t in lvlSet["levels"]:
lvlName = t[0]
lvlPath = t[1]
for i in range(len(lvlPath)):
lvlPath[i] = join(lvlSetDir, lvlPath[i])
error("Loading level: ", lvlPath[0])
lvl = loadBoogieFile(lvlPath[0])
lvl["path"] = lvlPath
if (len(t) > 2):
splitterPreds = [parseExprAst(exp) for exp in t[2]]
splitterPred = ast_and(splitterPreds)
remainderInv = parseExprAst(t[3])
lvl['data'][0] = [
row for row in lvl['data'][0]
if eval_quick(splitterPred,
{k: v
for k, v in zip(lvl['variables'], row)})
]
if (len(lvl['data'][0]) == 0):
error("SKIPPING : ", lvlName, " due to no filtered rows.")
continue
lvl['partialInv'] = remainderInv
lvl['splitterPreds'] = splitterPreds
lvls[lvlName] = lvl
return (lvlSet["name"], lvls)
def really_verified_by_play(lvl, assignment, worker, exp, play, timeout):
invs = set([
parseExprAst(x.payl()['canonical']) for x in play
if x.type == 'FoundInvariant'
])
(overfitted,
_), (nonind,
_), sound, violations = tryAndVerifyLvl(lvl, invs, set(), timeout)
assert type(violations) == list
return len(violations) == 0
def loadLvl(levelSet: str, lvlId: str, mturkId: MturkIdT, individualMode:bool = False) -> BoogieTraceLvl:
""" Load a given level. """
if (levelSet not in traces):
raise Exception("Unkonwn level set " + levelSet)
if (lvlId not in traces[levelSet]):
raise Exception("Unkonwn trace " + lvlId + " in levels " + levelSet)
lvl = traces[levelSet][lvlId]
fun: Function = lvl['program']
lvl = {
'lvlSet': levelSet,
'id': lvlId,
'variables': lvl['variables'],
'data': lvl['data'],
'typeEnv': typeEnvToJson(fun.getTypeEnv()),
'hint': lvl['hint'],
'goal': 'verify'
}
if args.colSwap:
nCols = len(lvl["variables"])
if nCols not in columnSwaps:
raise Exception("No column swap for %d columns" % nCols)
nSwaps = (nCols + 1) // 2 # swaps are 0 to nSwaps - 1 inclusive
session = sessionF()
colSwaps: List[int] = [0] * nSwaps
if individualMode:
workerId = mturkId[0]
allInvs(session, enames=[args.ename], lvlsets=[curLevelSetName],
lvls=[lvlId], workers=[workerId], colSwaps=colSwaps)
else:
allInvs(session, enames=[args.ename], lvlsets=[curLevelSetName],
lvls=[lvlId], colSwaps=colSwaps)
sortKeys = colSwaps
nSwap = sorted(zip(sortKeys, list(range(nSwaps))), key=lambda x: x[0])[0][1]
lvl["colSwap"] = nSwap
lvl["variables"] = swapColumns(lvl["variables"], nSwap)
lvl["data"] = [[swapColumns(row, nSwap) for row in rows]
for rows in lvl["data"]]
lvl["startingInvs"] = []
if args.replay:
# Only show players their own invariants, even if individual mode is off
invs = allInvs(session, enames=[args.ename], lvlsets=[curLevelSetName],
lvls=[lvlId], workers=[workerId])
lvl["startingInvs"] = [(inv[0], boogieToEsprima(parseExprAst(inv[1])))
for inv in invs]
return lvl
if (endsWithNumP.match(lvl_name)):
origName = justEnd.split(lvl_name)[0]
originalToSplitM[origName] = \
originalToSplitM.get(origName, []) + [ lvl_name ]
splitToOriginal[lvl_name] = origName
print("Level, #Splits, Solved, #FoundInvariants, #Added Implications, " +\
"#Added SPs, #Sound, #Overfit, #Nonind")
for origName in originalToSplitM:
found = reduce(
lambda x, y: x.union(y),
[lvlStats[z]["invariantsFound"] for z in originalToSplitM[origName]],
set())
implied = [ set([ bast.AstBinExpr(precc, "==>", bast.parseExprAst(inv[1]))
for inv in lvlStats[z]["invariantsFound"] \
for precc in lvls[z]["splitterPreds"] ])
for z in originalToSplitM[origName] ]
implied = reduce(lambda x, y: x.union(y), implied, set())
found = set([bast.parseExprAst(x[1]) for x in found])
nraw = len(found)
nimplied = len(implied)
lvl = lvls[originalToSplitM[origName][0]]
bbs = lvl["program"]
loop = lvl["loop"]
loop_header = loop.loop_paths[0][0]
sps = propagate_sp(bbs)[loop_header]
nsps = len(sps)
invs = found.union(sps).union(implied)
p.add_argument('--lvlid', type=str, \
default = 'desugared-boogie-benchmarks', \
help='Lvl-id in level set to try and verify"')
p.add_argument('invs', type=str, nargs="+", help="Invariants to try")
p.add_argument('--timeout', type=int, \
help="Optional z3 timeout", default=None)
args = p.parse_args();
_, lvls = loadBoogieLvlSet(args.lvlset)
lvl = lvls[args.lvlid]
boogie_invs = set([])
for inv in args.invs:
try:
bInv = parseExprAst(inv);
boogie_invs.add(bInv);
except Exception:
error("Failed parsing invariant " + inv);
exit(-1);
((overfitted, overfitted_p2), (nonind, nonind_p2), sound, violations) =\
tryAndVerifyLvl(lvl, boogie_invs, set(), args.timeout)
overfitted = set(overfitted).union(overfitted_p2)
nonind = set(nonind).union(nonind_p2)
print("Overfitted: ", overfitted)
print("Nonind: ", nonind)
print("Sound: ", sound)
print("Violations: ", violations)
def tryAndVerify(
levelSet: str,
levelId: str,
invs: List[EsprimaNode],
mturkId: MturkIdT,
individualMode: bool
) -> Tuple[List[Tuple[EsprimaNode, StoreArr]],\
List[Tuple[EsprimaNode, Tuple[StoreArr, StoreArr]]],\
List[EsprimaNode],\
List[StoreArr],
List[StoreArr]]:
"""
Given a level (levelSet, levelId) and a set of invaraints invs do:
1) Find all invariants OldI that were not DEFINITELY false from the
last time we tried to verify this level (getLastVerResult)
2) Try and verify the level using S = OldI + invs. This is done by
calling tryAndVerifyLvl. For more on that see comment in
tryAndVerifyLvl
3) Return object containing:
overfitted - all invariants in invs not implied by precondition
nonind - all invariants in invs that are not inductive
sound - all invariants in invs that are sound
post_ctrex - any safety counterexamples to the "sound" invariants.
If this set is empty, then the level is verified.
direct_ctrex - any safety counterexamples to ALL of the passed in
invs (not just the sound ones). This is used by the 'rounds' game
mode to generate red rows for the next level.
"""
s = sessionF()
if (levelSet not in traces):
raise Exception("Unkonwn level set " + str(levelSet))
if (levelId not in traces[levelSet]):
raise Exception("Unkonwn trace " + str(levelId) + \
" in levels " + str(levelSet))
if (len(invs) == 0):
raise Exception("No invariants given")
lvl: BoogieTraceLvl = traces[levelSet][levelId]
assert 'program' in lvl
workerId, _, _ = mturkId
userInvs = set([ esprimaToBoogie(x, {}) for x in invs ])
otherInvs: Set[AstExpr] = set([])
lastVer: Optional[Dict[str, Any]] = None
#lastVer = getLastVerResult(levelSet, levelId, s,
# workerId=(workerId if individualMode else None))
if (lastVer):
otherInvs = otherInvs.union([parseExprAst(x) for x in lastVer["sound"]])
otherInvs = otherInvs.union([parseExprAst(x) for x in lastVer["nonind"]])
((overfitted, _), (nonind, _), sound, violations) = \
tryAndVerifyLvl(lvl, userInvs, otherInvs, 1) # making all calls 1 sec args.timeout)
# See if the level is solved
solved = len(violations) == 0
fix = lambda x: _from_dict(lvl['variables'], x, 0)
# TODO: Why is this shit needed?
if (not solved):
fun: Function = lvl["program"]
direct_ctrexs = loopInvSafetyCtrex(fun, otherInvs.union(userInvs),\
1) # making all calls 1 sec args.timeout)
else:
direct_ctrexs = []
# Convert all invariants from Boogie to esprima expressions, and
# counterexamples to arrays
# from dictionaries
overfittedEsp = [ (boogieToEsprima(inv), fix(v.endEnv()))
for (inv, v) in overfitted ]
nonindEsp = [ (boogieToEsprima(inv), (fix(v.startEnv()), fix(v.endEnv())))
for (inv, v) in nonind ]
soundEsp = [ boogieToEsprima(inv) for inv in sound ]
safetyCtrexsArr = [ fix(v.startEnv()) for v in violations ]
directCtrexsArr = [ fix(v) for v in direct_ctrexs ]
res = (overfittedEsp, nonindEsp, soundEsp, safetyCtrexsArr, directCtrexsArr)
# Log verification result in Db (storing invariants as boogie strings)
payl = {
"lvlset": levelSet,
"lvlid": levelId,
"overfitted":nodups([str(inv) for (inv, _) in overfitted]),
"nonind":nodups([str(inv) for (inv,_) in nonind]),
"sound":nodups([str(inv) for inv in sound]),
"post_ctrex":safetyCtrexsArr,
"direct_ctrex": directCtrexsArr
}
addEvent("verifier", "VerifyAttempt", time(), args.ename, \
"localhost", payl, s, mturkId)
return res
def processLevel(args,
lvl,
lvls=None,
lvlName=None,
additionalInvs=[],
storeArgs=None,
worker=None,
assignment=None):
workers = args.workers if worker is None else [worker]
assignments = None if assignment is None else [assignment]
sessionF = open_db(args.db)
s = sessionF()
actualEnames = set()
actualLvls = set()
actualLvlsets = set()
actualWorkers = set()
actualAssignments = set()
dbInvs = allInvs(s,
enames=args.enames,
lvls=lvls,
lvlsets=args.lvlsets,
workers=workers,
assignments=assignments,
enameSet=actualEnames,
lvlSet=actualLvls,
lvlsetSet=actualLvlsets,
workerSet=actualWorkers,
assignmentSet=actualAssignments)
invs = set(parseExprAst(inv[1]) for inv in dbInvs)
invs.update(additionalInvs)
if lvlName is not None:
print("_" * 40)
print("RESULTS FOR LEVEL", lvlName)
print()
print("ENAMES:", ", ".join(actualEnames))
print("LVLS:", ", ".join(actualLvls))
print("LVLSETS:", ", ".join(actualLvlsets))
if len(actualWorkers) < 6:
print("WORKERS:", ", ".join(actualWorkers))
else:
print("UNIQUE WORKERS:", len(actualWorkers))
if len(actualAssignments) < 3:
print("ASSIGNMENTS:", ", ".join(actualAssignments))
else:
print("UNIQUE ASSIGNMENTS:", len(actualAssignments))
if len(additionalInvs) < 6:
print("ADDITIONAL INVARIANTS:",
", ".join(str(x) for x in additionalInvs))
else:
print("ADDITIONAL INVARIANTS:", len(additionalInvs))
if len(invs) < 6:
print("UNIQUE INVARIANTS:", ", ".join(map(str, invs)))
else:
print("UNIQUE INVARIANTS:", len(invs))
print()
payload = {
"enames": list(actualEnames),
"lvls": list(actualLvls),
"lvlsets": list(actualLvlsets),
"workers": list(actualWorkers),
"assignments": list(actualAssignments)
}
overfittedSet = set()
nonindSet = set()
soundSet = set()
proved = verify(lvl,
invs,
timeout=args.timeout,
overfittedSet=overfittedSet,
nonindSet=nonindSet,
soundSet=soundSet)
payload["overfitted"] = list(str(s) for s in overfittedSet)
payload["nonind"] = list(str(s) for s in nonindSet)
payload["sound"] = list(str(s) for s in soundSet)
if storeArgs is not None:
config, lvlset, now = storeArgs
storeVerify(s, config, lvlset, lvlName, args.timeout, now, proved,
payload)
print("Either --bpl or --lvlset must be specified")
exit(1)
additionalInvs = {}
if args.additionalInvs:
with open(args.additionalInvs) as f:
r = csv.reader(f, delimiter=",")
for row in r:
if not row:
continue
invs = []
for invstr in invlist.split(";"):
if not len(invstr.strip()):
continue
try:
inv = parseExprAst(invstr)
if tautology(expr_to_z3(inv, AllIntTypeEnv())):
print(
"Dropping additional invariant (tautology): ",
inv)
continue
except RuntimeError:
# Some invariants are too large to parse
print("Dropping additional invariant (parse): ", inv)
continue
except Unknown:
# Timeouts could be valid invariants
pass
invs.append(inv)
additionalInvs[lvlName] = invs
"Not Proved"))
print("-- Worker ID: " + worker_id + \
( "" if worker_id == assn_worker_id else \
" (NOTE: different from worker ID in assignment)"))
print("-- IP: " + ip)
print("-- Time when finished: " + time_str)
print("-- User invs: " + ", ".join(js_invs))
boogie_user_invs = [ esprimaToBoogie(x, {}) \
for x in canon_invs ]
try:
solName = os.path.join(get_lvlset_dir(lvl_set), \
lvl_id + ".soln")
with open(solName) as f:
for l in f:
boogie_soln_inv = ast.parseExprAst(l)
header = "-- Soln " + str(boogie_soln_inv) + \
": "
found = False
for boogie_user_inv in boogie_user_invs:
if equiv(boogie_soln_inv, boogie_user_inv):
print(header + "Found as user " +\
"predicate (canon version): " + \
str(boogie_user_inv))
found = True
if not found:
print(header + "No equiv found")
except IOError:
print("-- No .soln file")
if event_name == "GameDone":
print("Missing C path for ", lvl_name)
else:
if (not(exists(paths[1]))):
print("C file for ", lvl_name, ":", paths[1], "is missing")
if (len(paths) == 0):
print("Missing desugared boogie file path for", lvl_name);
else:
if (not exists(paths[0])):
print("Desugared boogie for ", lvl_name, ":", paths[0], "is missing");
print("Checking all solutions work...")
for lvl_name, lvl in list(lvls.items()):
sol = open(lvl["path"][0].replace(".bpl", ".sol")).read()
sol = bast.parseExprAst(sol);
bbs = lvl["program"]
loop = lvl["loop"]
try:
(overfit, nonind, sound, violations) = verify(lvl, [sol])
if (not len(violations) == 0):
print(lvl_name, "doesn't satisfy solution ", sol, "details:", \
(overfit, nonind, sound, violations))
except Unknown:
print("Can't tell for level ", lvl_name)
continue
endsWithNumP = reComp(".*\.[0-9]*$")
justEnd = reComp("\.[0-9]*$")
def parseInvGenInvariant(inv):
replaceEq = re.compile("([^<>=])=([^<>=])")
inv = replaceEq.sub(r"\1==\2", inv)
inv = inv.replace("=<", "<=")
return parseExprAst(inv)