def getSolutions(): # Lvlset is assumed to be current by default
res = {}
for lvlId in lvls:
solnFile = lvls[lvlId]["path"][0][:-len(".bpl")] + ".sol"
soln = open(solnFile).read().strip()
boogieSoln = parseExprAst(soln)
res[curLevelSetName + "," + lvlId] = [boogieToEsprimaExpr(boogieSoln)]
return res
def bad_envs_to_expr(bad_envs):
s = "&&".join(["!(" + \
"&&".join([("(%s==%s)" %(k, str(v))) \
for (k,v) in bad_env.iteritems()]) + \
")"
for bad_env in bad_envs])
if (s == ""):
return AstTrue()
return parseExprAst(s)
def loadBoogieLvlSet(lvlSetFile):
# Small helper funct to make sure we didn't
# accidentally give two levels the same name
def assertUniqueKeys(kvs):
keys = [x[0] for x in kvs]
assert (len(set(keys)) == len(keys))
return dict(kvs)
lvlSet = load(open(lvlSetFile, "r"), object_pairs_hook=assertUniqueKeys)
lvlSetDir = dirname(abspath(realpath(lvlSetFile)))
error("Loading level set " + lvlSet["name"] + " from " + lvlSetFile)
lvls = OrderedDict()
for t in lvlSet["levels"]:
lvlName = t[0]
lvlPath = t[1]
for i in range(len(lvlPath)):
lvlPath[i] = join(lvlSetDir, lvlPath[i])
error("Loading level: ", lvlPath[0])
lvl = loadBoogieFile(lvlPath[0], False)
lvl["path"] = lvlPath
if (len(t) > 2):
splitterPreds = [parseExprAst(exp) for exp in t[2]]
splitterPred = ast_and(splitterPreds)
remainderInv = parseExprAst(t[3])
lvl['data'][0] = filter(
lambda row: evalPred(splitterPred,
_to_dict(lvl['variables'], row)),
lvl['data'][0])
if (len(lvl['data'][0]) == 0):
error("SKIPPING : ", lvlName, " due to no filtered rows.")
continue
lvl['partialInv'] = remainderInv
lvl['splitterPreds'] = splitterPreds
lvls[lvlName] = lvl
return (lvlSet["name"], lvls)
def really_verified_by_play(lvl, assignment, worker, exp, play, timeout):
invs = set([
parseExprAst(x.payl()['canonical']) for x in play
if x.type == 'FoundInvariant'
])
(overfitted,
_), (nonind,
_), sound, violations = tryAndVerifyLvl(lvl, invs, set(), timeout)
assert type(violations) == list
return len(violations) == 0
def processLevel(args,
lvl,
lvls=None,
lvlName=None,
additionalInvs=[],
storeArgs=None,
worker=None,
assignment=None):
workers = args.workers if worker is None else [worker]
assignments = None if assignment is None else [assignment]
sessionF = open_db(args.db)
s = sessionF()
actualEnames = set()
actualLvls = set()
actualLvlsets = set()
actualWorkers = set()
actualAssignments = set()
dbInvs = allInvs(s,
enames=args.enames,
lvls=lvls,
lvlsets=args.lvlsets,
workers=workers,
assignments=assignments,
enameSet=actualEnames,
lvlSet=actualLvls,
lvlsetSet=actualLvlsets,
workerSet=actualWorkers,
assignmentSet=actualAssignments)
invs = set(parseExprAst(inv[1]) for inv in dbInvs)
invs.update(additionalInvs)
if lvlName is not None:
print "_" * 40
print "RESULTS FOR LEVEL", lvlName
print
print "ENAMES:", ", ".join(actualEnames)
print "LVLS:", ", ".join(actualLvls)
print "LVLSETS:", ", ".join(actualLvlsets)
if len(actualWorkers) < 6:
print "WORKERS:", ", ".join(actualWorkers)
else:
print "UNIQUE WORKERS:", len(actualWorkers)
if len(actualAssignments) < 3:
print "ASSIGNMENTS:", ", ".join(actualAssignments)
else:
print "UNIQUE ASSIGNMENTS:", len(actualAssignments)
if len(additionalInvs) < 6:
print "ADDITIONAL INVARIANTS:", ", ".join(
str(x) for x in additionalInvs)
else:
print "ADDITIONAL INVARIANTS:", len(additionalInvs)
if len(invs) < 6:
print "UNIQUE INVARIANTS:", ", ".join(map(str, invs))
else:
print "UNIQUE INVARIANTS:", len(invs)
print
payload = {
"enames": list(actualEnames),
"lvls": list(actualLvls),
"lvlsets": list(actualLvlsets),
"workers": list(actualWorkers),
"assignments": list(actualAssignments)
}
overfittedSet = set()
nonindSet = set()
soundSet = set()
proved = verify(lvl,
invs,
timeout=args.timeout,
overfittedSet=overfittedSet,
nonindSet=nonindSet,
soundSet=soundSet)
payload["overfitted"] = list(str(s) for s in overfittedSet)
payload["nonind"] = list(str(s) for s in nonindSet)
payload["sound"] = list(str(s) for s in soundSet)
if storeArgs is not None:
config, lvlset, now = storeArgs
storeVerify(s, config, lvlset, lvlName, args.timeout, now, proved,
payload)
print "Either --bpl or --lvlset must be specified"
exit(1)
additionalInvs = {}
if args.additionalInvs:
with open(args.additionalInvs) as f:
r = csv.reader(f, delimiter=",")
for row in r:
if not row:
continue
invs = []
for invstr in invlist.split(";"):
if not len(invstr.strip()):
continue
try:
inv = parseExprAst(invstr)
if tautology(expr_to_z3(inv, AllIntTypeEnv())):
print "Dropping additional invariant (tautology): ", inv
continue
except RuntimeError:
# Some invariants are too large to parse
print "Dropping additional invariant (parse): ", inv
continue
except Unknown:
# Timeouts could be valid invariants
pass
invs.append(inv)
additionalInvs[lvlName] = invs
print "ADDITIONAL INVARIANTS LOADED FOR LVLS:", \
", ".join(additionalInvs.keys())
if __name__ == "__main__":
args = p.parse_args();
s = open_sqlite_db("../logs/" + args.ename + "/events.db")()
lvlsetName, lvls = loadBoogieLvlSet(args.lvlset)
otherInvs = { }
if (args.additionalInvs):
with open(args.additionalInvs) as f:
r = csv.reader(f, delimiter=",");
for row in r:
(lvl, invs) = row
bInvs = []
for inv in [x for x in invs.split(";") if len(x.strip()) != 0]:
try:
bInv = parseExprAst(inv)
if (tautology(expr_to_z3(bInv, AllIntTypeEnv()))):
continue
bInvs.append(bInv)
except RuntimeError:
# Some invariants are just too large for parsing :(
pass
except Unknown:
bInvs.append(bInv)
otherInvs[lvl]=bInvs
lvlStats = { lvlN: {
"usersStarted": set(),\
"nusersStarted": 0,\
"usersFinished": set(),\
def loadBoogieFile(fname, multiround):
bbs = get_bbs(fname)
ensureSingleExit(bbs)
loop = unique(loops(bbs),
"Cannot handle program with multiple loops:" + fname)
# The variables to trace are all live variables at the loop header
vs = list(livevars(bbs)[loop.loop_paths[0][0]])
# Make sure variable names are different modulo case
assert len(vs) == len(set([var.lower() for var in vs]))
# See if there is a .trace or a .hint file
hint = None
header_vals = None
terminates = False
try:
(vs, header_vals) = readTrace(fname[:-4] + '.trace')
hint = load(open(fname[:-4] + '.hint'))
except Exception: #TODO (Dimo) IOError here instead?
pass
if (not header_vals):
header_vals, terminates = _tryUnroll(loop, bbs, 0, 4, None, None)
# Assume we have no tests with dead loops
assert (header_vals != [])
invTemplates = ["_sv_x<_sv_y", "_sv_x<=_sv_y", "_sv_x==_sc_c", \
"_sv_x==_sv_y", "_sv_x==0", "_sv_x<0"]
invTemplates = [parseExprAst(inv) for inv in invTemplates]
new_header_vals, new_terminates = \
getInitialData(loop, bbs, 4, invTemplates, [ "_sv_x", "_sv_y" ])
if (new_header_vals != None):
header_vals = new_header_vals
terminates = new_terminates
writeTrace(fname[:-4] + ".trace", new_header_vals)
return {
'variables':
vs,
'data': [[[str(row[v]) for v in vs] for row in header_vals], [], []],
'exploration_state':
[([str(header_vals[0][v]) for v in vs], len(header_vals), terminates)],
'hint':
hint,
'goal': {
"verify": True
},
'support_pos_ex':
True,
'support_neg_ex':
True,
'support_ind_ex':
True,
'multiround':
multiround,
'program':
bbs,
'loop':
loop
}
p.add_argument('--lvlid', type=str, \
default = 'desugared-boogie-benchmarks', \
help='Lvl-id in level set to try and verify"')
p.add_argument('invs', type=str, nargs="+", help="Invariants to try")
p.add_argument('--timeout', type=int, \
help="Optional z3 timeout", default=None)
args = p.parse_args();
_, lvls = loadBoogieLvlSet(args.lvlset)
lvl = lvls[args.lvlid]
boogie_invs = set([])
for inv in args.invs:
try:
bInv = parseExprAst(inv);
boogie_invs.add(bInv);
except Exception:
error("Failed parsing invariant " + inv);
exit(-1);
((overfitted, overfitted_p2), (nonind, nonind_p2), sound, violations) =\
tryAndVerifyLvl(lvl, boogie_invs, set(), args.timeout)
overfitted = set(overfitted).union(overfitted_p2)
nonind = set(nonind).union(nonind_p2)
print "Overfitted: ", overfitted
print "Nonind: ", nonind
print "Sound: ", sound
print "Violations: ", violations
def tryAndVerify(levelSet, levelId, invs, mturkId, individualMode):
"""
Given a level (levelSet, levelId) and a set of invaraints invs do:
1) Find all invariants OldI that were not DEFINITELY false from the
last time we tried to verify this level (getLastVerResult)
2) Try and verify the level using S = OldI + invs. This is done by
calling tryAndVerifyLvl. For more on that see comment in
tryAndVerifyLvl
3) Return object containing:
overfitted - all invariants in invs not implied by precondition
nonind - all invariants in invs that are not inductive
sound - all invariants in invs that are sound
post_ctrex - any safety counterexamples to the "sound" invariants.
If this set is empty, then the level is verified.
direct_ctrex - any safety counterexamples to ALL of the passed in
invs (not just the sound ones). This is used by the 'rounds' game
mode to generate red rows for the next level.
"""
s = sessionF()
if (levelSet not in traces):
raise Exception("Unkonwn level set " + str(levelSet))
if (levelId not in traces[levelSet]):
raise Exception("Unkonwn trace " + str(levelId) + \
" in levels " + str(levelSet))
if (len(invs) == 0):
raise Exception("No invariants given")
lvl = traces[levelSet][levelId]
if ('program' not in lvl):
# Not a boogie level - error
raise Exception("Level " + str(levelId) + " " + \
str(levelSet) + " not a dynamic boogie level.")
workerId, _, _ = mturkId
print repr(set)
userInvs = set([esprimaToBoogie(x, {}) for x in invs])
otherInvs = set([])
lastVer = getLastVerResult(levelSet,
levelId,
s,
workerId=(workerId if individualMode else None))
if (lastVer):
otherInvs = otherInvs.union(
[parseExprAst(x) for x in lastVer["sound"]])
otherInvs = otherInvs.union(
[parseExprAst(x) for x in lastVer["nonind"]])
((overfitted, _), (nonind, _), sound, violations) =\
tryAndVerifyLvl(lvl, userInvs, otherInvs, args.timeout)
# See if the level is solved
solved = len(violations) == 0
fix = lambda x: _from_dict(lvl['variables'], x, 0)
if (not solved):
bbs = lvl["program"]
loop = lvl["loop"]
direct_ctrexs = loopInvSafetyCtrex(loop, otherInvs.union(userInvs),\
bbs, args.timeout)
else:
direct_ctrexs = []
# Convert all invariants from Boogie to esprima expressions, and
# counterexamples to arrays
# from dictionaries
overfitted = [(boogieToEsprima(inv), fix(v.endEnv()))
for (inv, v) in overfitted]
nonind = [(boogieToEsprima(inv), (fix(v.startEnv()), fix(v.endEnv())))
for (inv, v) in nonind]
sound = [boogieToEsprima(inv) for inv in sound]
safety_ctrexs = [fix(v.startEnv()) for v in violations]
direct_ctrexs = [fix(v) for v in direct_ctrexs]
res = (overfitted, nonind, sound, safety_ctrexs, direct_ctrexs)
payl = {
"lvlset": levelSet,
"lvlid": levelId,
"overfitted":
nodups([str(esprimaToBoogie(x[0], {})) for x in overfitted]),
"nonind":
nodups([str(esprimaToBoogie(inv, {})) for (inv, _) in nonind]),
"sound": nodups([str(esprimaToBoogie(inv, {})) for inv in sound]),
"post_ctrex": safety_ctrexs,
"direct_ctrex": direct_ctrexs
}
addEvent("verifier", "VerifyAttempt", time(), args.ename, \
"localhost", payl, s, mturkId)
return res
def genNextLvl(workerId, mturkId, levelSet, levelId, invs, individualMode):
""" Given a level (levelSet, levelId) and a set of invariants invs
attempted by a user, generate and return a new level by appending the
counterexamples to invs to the current level. The new level has the
same id as levelId with ".g" appended at the end. This is a hack.
"""
s = sessionF()
if (levelSet not in traces):
raise Exception("Unkonwn level set " + str(levelSet))
if (levelId not in traces[levelSet]):
raise Exception("Unkonwn trace " + str(levelId) + \
" in levels " + str(levelSet))
if (len(invs) == 0):
raise Exception("No invariants given")
lvl = traces[levelSet][levelId]
if ('program' not in lvl):
# Not a boogie level - error
raise Exception("Level " + str(levelId) + " " + \
str(levelSet) + " not a dynamic boogie level.")
userInvs = set([esprimaToBoogie(x, {}) for x in invs])
otherInvs = set([])
lastSoundInvs = set([])
lastNonindInvs = set([])
lastVer = getLastVerResult(levelSet,
levelId,
s,
workerId=(workerId if individualMode else None))
if (lastVer):
lastSoundInvs = set([parseExprAst(x) for x in lastVer["sound"]])
lastNonindInvs = set([parseExprAst(x) for x in lastVer["nonind"]])
otherInvs = lastSoundInvs.union(lastNonindInvs)
((overfitted, _), (_, _), sound, violations) =\
tryAndVerifyLvl(lvl, userInvs, otherInvs, args.timeout)
# See if the level is solved
solved = len(violations) == 0
if (solved):
payl = [
levelSet, levelId, invs, [boogieToEsprimaExpr(e) for e in sound]
]
addEvent("verifier", "GenNext.Solved", time(), args.ename, \
"localhost", payl, s, mturkId)
return loadNextLvl(workerId, mturkId, individualMode)
fix = lambda env: _from_dict(lvl['variables'], env, 0)
greenRows = [fix(v.endEnv()) for v in overfitted if type(v) != tuple]
print "Invs: ", otherInvs.union(userInvs)
print "GreenRows: ", greenRows
bbs = lvl["program"]
loop = lvl["loop"]
ctrexInvs = lastSoundInvs.union(userInvs)
safetyCtrex =\
loopInvSafetyCtrex(loop, ctrexInvs, bbs, args.timeout)
redRows = [fix(x) for x in safetyCtrex if len(x) != 0]
print "RedRows: ", redRows
if (len(redRows) > 0 or len(greenRows) > 0):
# Lets give them another level
payl = [
levelSet, levelId, invs,
[boogieToEsprimaExpr(e) for e in ctrexInvs]
]
addEvent("verifier", "GenNext.NoNewRows", time(), args.ename, \
"localhost", payl, s, mturkId)
newLvl = copy(lvl)
newLvlId = levelId + ".g"
newLvl["data"][0].extend(greenRows)
newLvl["data"][2].extend(redRows)
traces[levelSet][newLvlId] = newLvl
return loadLvl(levelSet, newLvlId, mturkId, individualMode)
else:
# Else give them the actual next level
return loadNextLvl(workerId, mturkId, individualMode)
def loadLvl(levelSet, lvlId, mturkId, individualMode=False): #pylint: disable=unused-argument
""" Load a given level. """
if (levelSet not in traces):
raise Exception("Unkonwn level set " + levelSet)
if (lvlId not in traces[levelSet]):
raise Exception("Unkonwn trace " + lvlId + " in levels " + levelSet)
lvl = traces[levelSet][lvlId]
if ('program' in lvl):
# This is a boogie level - don't return the program/loop and other book
# keeping
lvl = {
'lvlSet': levelSet,
'id': lvlId,
'variables': lvl['variables'],
'data': lvl['data'],
'exploration_state': lvl['exploration_state'],
'hint': lvl['hint'],
'goal': lvl['goal'],
'support_pos_ex': lvl['support_pos_ex'],
'support_neg_ex': lvl['support_neg_ex'],
'support_ind_ex': lvl['support_ind_ex'],
'multiround': lvl['multiround'],
}
if args.colSwap:
nCols = len(lvl["variables"])
if nCols not in columnSwaps:
raise Exception("No column swap for %d columns" % nCols)
nSwaps = (nCols + 1) // 2 # swaps are 0 to nSwaps - 1 inclusive
session = sessionF()
colSwaps = [0] * nSwaps
allInvs(session,
enames=[args.ename],
lvlsets=[curLevelSetName],
lvls=[lvlId],
colSwaps=colSwaps)
sortKeys = colSwaps
if individualMode:
workerId = mturkId[0]
colSwaps = [0] * nSwaps
allInvs(session,
enames=[args.ename],
lvlsets=[curLevelSetName],
lvls=[lvlId],
workers=[workerId],
colSwaps=colSwaps)
sortKeys = zip(colSwaps, sortKeys)
nSwap = sorted(zip(sortKeys, range(nSwaps)), key=lambda x: x[0])[0][1]
lvl["colSwap"] = nSwap
lvl["variables"] = swapColumns(lvl["variables"], nSwap)
lvl["data"] = [[swapColumns(row, nSwap) for row in rows]
for rows in lvl["data"]]
lvl["startingInvs"] = []
if args.replay:
# Only show players their own invariants, even if individual mode is off
invs = allInvs(session,
enames=[args.ename],
lvlsets=[curLevelSetName],
lvls=[lvlId],
workers=[workerId])
lvl["startingInvs"] = [(inv[0], boogieToEsprima(parseExprAst(inv[1])))
for inv in invs]
return lvl
def parseInvGenInvariant(inv):
replaceEq = re.compile("([^<>=])=([^<>=])")
inv = replaceEq.sub(r"\1==\2", inv)
inv = inv.replace("=<", "<=")
return parseExprAst(inv)
print "Missing C path for ", lvl_name
else:
if (not (exists(paths[1]))):
print "C file for ", lvl_name, ":", paths[1], "is missing"
if (len(paths) == 0):
print "Missing desugared boogie file path for", lvl_name
else:
if (not exists(paths[0])):
print "Desugared boogie for ", lvl_name, ":", paths[
0], "is missing"
print "Checking all solutions work..."
for lvl_name, lvl in lvls.items():
sol = open(lvl["path"][0].replace(".bpl", ".sol")).read()
sol = bast.parseExprAst(sol)
bbs = lvl["program"]
loop = lvl["loop"]
try:
(overfit, nonind, sound, violations) = verify(lvl, [sol])
if (not len(violations) == 0):
print lvl_name, "doesn't satisfy solution ", sol, "details:", \
(overfit, nonind, sound, violations)
except Unknown:
print "Can't tell for level ", lvl_name
continue
endsWithNumP = reComp(".*\.[0-9]*$")
justEnd = reComp("\.[0-9]*$")
if (endsWithNumP.match(lvl_name)):
origName = justEnd.split(lvl_name)[0]
originalToSplitM[origName] = \
originalToSplitM.get(origName, []) + [ lvl_name ]
splitToOriginal[lvl_name] = origName
print "Level, #Splits, Solved, #FoundInvariants, #Added Implications, " +\
"#Added SPs, #Sound, #Overfit, #Nonind"
for origName in originalToSplitM:
found = reduce(
lambda x, y: x.union(y),
[lvlStats[z]["invariantsFound"] for z in originalToSplitM[origName]],
set())
implied = [ set([ bast.AstBinExpr(precc, "==>", bast.parseExprAst(inv[1]))
for inv in lvlStats[z]["invariantsFound"] \
for precc in lvls[z]["splitterPreds"] ])
for z in originalToSplitM[origName] ]
implied = reduce(lambda x, y: x.union(y), implied, set())
found = set([bast.parseExprAst(x[1]) for x in found])
nraw = len(found)
nimplied = len(implied)
lvl = lvls[originalToSplitM[origName][0]]
bbs = lvl["program"]
loop = lvl["loop"]
loop_header = loop.loop_paths[0][0]
sps = propagate_sp(bbs)[loop_header]
nsps = len(sps)
invs = found.union(sps).union(implied)
"Not Proved")
print "-- Worker ID: " + worker_id + \
( "" if worker_id == assn_worker_id else \
" (NOTE: different from worker ID in assignment)")
print "-- IP: " + ip
print "-- Time when finished: " + time_str
print "-- User invs: " + ", ".join(js_invs)
boogie_user_invs = [ esprimaToBoogie(x, {}) \
for x in canon_invs ]
try:
solName = os.path.join(get_lvlset_dir(lvl_set), \
lvl_id + ".soln")
with open(solName) as f:
for l in f:
boogie_soln_inv = ast.parseExprAst(l)
header = "-- Soln " + str(boogie_soln_inv) + \
": "
found = False
for boogie_user_inv in boogie_user_invs:
if equiv(boogie_soln_inv, boogie_user_inv):
print header + "Found as user " +\
"predicate (canon version): " + \
str(boogie_user_inv)
found = True
if not found:
print header + "No equiv found"
except IOError:
print "-- No .soln file"
if event_name == "GameDone":