def analyzeExperiments(self, explist, skip=1):
balancedb = {}
proc = runBashScript("""
cd %(expdir)s
for TESTDIR in %(testdirs)s; do
find ./$TESTDIR -name 'out-lb.txt' | while read LINE; do
echo $LINE
cat $LINE | grep "IN TOTAL" | sed -n '1~%(skip)d p; $ p;'
cat $LINE | grep "Created transfer request"
done
done""" % {
"testdirs": " ".join(explist),
"expdir": self.localhost["expdir"],
"skip": skip
},
stdout=PIPE)
data, _ = proc.communicate()
lines = data.splitlines()
for line in lines:
match = self.pathRe.match(line)
if match:
testdir, target, workercount, _, tgcount = match.groups()
workercount = int(workercount)
tgcount = int(tgcount) if tgcount else 1
tgdata = balancedb.setdefault(target, {})
expdata = tgdata.setdefault((testdir, tgcount), {})
timeline = expdata.setdefault(workercount, BalanceTimeLine())
continue
match = self.totalRe.match(line)
if match:
timestamp, total = float(match.group(1)), int(match.group(2))
timeline.totals.append((timestamp, total))
continue
match = self.transferRe.match(line)
if match:
timestamp = float(match.group(1))
amount = int(match.group(4))
timeline.transfers.append((timestamp, amount))
if amount > 0:
if timeline.transfers and abs(timeline.transfers[-1][0] -
timestamp) < 0.01:
timeline.transfers[-1] = (timestamp, amount +
timeline.transfers[-1][1])
else:
timeline.transfers.append((timestamp, amount))
continue
self._logMsg("Unprocessed line: '%s'" % line)
avgdb = self._averageData(balancedb)
return avgdb
def _prepareRemoteHost(self, host, cleanCores=True):
proc = runBashScript("""
ssh -o StrictHostKeyChecking=no %(user)s@%(host)s 'bash -s' <<EOF && \
scp %(coverage)s %(user)s@%(host)s:%(expdir)s/%(newdir)s/$(basename %(coverage)s)
# The code below is run remotely
if [ ! -f %(root)s/%(worker)s ]; then echo "Cannot find the Cloud9 worker executable: %(root)s/%(worker)s"; exit 1; fi
if [ ! -f %(root)s/%(klee)s ]; then echo "Cannot find the Klee executable: %(root)s/%(klee)s"; exit 1; fi
mkdir -p %(expdir)s/%(newdir)s
%(cleancores)s && find %(expdir)s -name 'core' | xargs rm -f
if [ -h %(expdir)s/last ]; then rm -f %(expdir)s/last; fi
[ ! -a %(expdir)s/last ] && ln -s %(expdir)s/%(newdir)s %(expdir)s/last
\nEOF""" % {
"user": self.hosts[host]["user"],
"host": host,
"coverage": self.coverable,
"root": self.hosts[host]["root"],
"worker": WORKER_PATH,
"klee": KLEE_PATH,
"expdir": self.hosts[host]["expdir"],
"newdir": self.uid,
"cleancores": "true" if cleanCores else "false"
})
if proc.wait() != 0:
self._logMsg("Unable to initialize host '%s'. Aborting..." % host)
exit(1)
else:
self._logMsg("Initialization complete for host '%s'." % host)
def analyzeExperiments(self, explist, skip=1):
balancedb = {}
proc = runBashScript(
"""
cd %(expdir)s
for TESTDIR in %(testdirs)s; do
find ./$TESTDIR -name 'out-lb.txt' | while read LINE; do
echo $LINE
cat $LINE | grep "IN TOTAL" | sed -n '1~%(skip)d p; $ p;'
cat $LINE | grep "Created transfer request"
done
done"""
% {"testdirs": " ".join(explist), "expdir": self.localhost["expdir"], "skip": skip},
stdout=PIPE,
)
data, _ = proc.communicate()
lines = data.splitlines()
for line in lines:
match = self.pathRe.match(line)
if match:
testdir, target, workercount, _, tgcount = match.groups()
workercount = int(workercount)
tgcount = int(tgcount) if tgcount else 1
tgdata = balancedb.setdefault(target, {})
expdata = tgdata.setdefault((testdir, tgcount), {})
timeline = expdata.setdefault(workercount, BalanceTimeLine())
continue
match = self.totalRe.match(line)
if match:
timestamp, total = float(match.group(1)), int(match.group(2))
timeline.totals.append((timestamp, total))
continue
match = self.transferRe.match(line)
if match:
timestamp = float(match.group(1))
amount = int(match.group(4))
timeline.transfers.append((timestamp, amount))
if amount > 0:
if timeline.transfers and abs(timeline.transfers[-1][0] - timestamp) < 0.01:
timeline.transfers[-1] = (timestamp, amount + timeline.transfers[-1][1])
else:
timeline.transfers.append((timestamp, amount))
continue
self._logMsg("Unprocessed line: '%s'" % line)
avgdb = self._averageData(balancedb)
return avgdb
def _killAllLocal(self, signal="SIGINT"):
proc = runBashScript("""
killall -%(signal)s %(worker)s
killall -%(signal)s %(klee)s
killall -%(signal)s %(lb)s""" % {
"signal": signal,
"worker": os.path.basename(WORKER_PATH),
"lb": os.path.basename(LB_PATH),
"klee": os.path.basename(KLEE_PATH)
})
proc.wait()
def _runWorker(self, host, port, lbHost, lbPort, target, workerID, workerCount, targetcounter):
logdir = "%s/%s" % (
self.localhost["expdir"],
self._getExperimentID(target, workerCount, targetcounter))
if self.cmdlines[target].startswith("/"):
cmdline = self.cmdlines[target]
else:
cmdline = "%s/%s" % (self.hosts[host]["targetdir"], self.cmdlines[target])
proc = runBashScript("""
mkdir -p %(logdir)s
ssh -o StrictHostKeyChecking=no %(user)s@%(host)s 'bash -s' <<EOF &>%(logfile)s
# The code below is run remotely
mkdir -p %(expdir)s
cd %(expdir)s
ulimit -c unlimited
setarch $(arch) -R %(root)s/%(worker)s -c9-lb-host %(lbhost)s -c9-lb-port %(lbport)d \
-c9-local-host %(lhost)s -c9-local-port %(lport)d %(jobsel)s \
-output-dir %(outdir)s \
%(kcmd)s %(debugcomm)s %(debugcov)s \
--max-time %(maxtime)d --coverable-modules %(coverable)s \
%(cmdline)s
\nEOF""" % {
"user": self.hosts[host]["user"],
"host": host,
"expdir": "%s/%s" % (
self.hosts[host]["expdir"],
self._getExperimentID(target, workerCount, targetcounter)),
"root": self.hosts[host]["root"],
"worker": WORKER_PATH,
"lbhost": lbHost,
"lbport": lbPort,
"lhost": host,
"lport": port,
"jobsel": " ".join(["-c9-job-%s" % x for x in
(self.strategy.split(",")
if self.strategy
else ["random-path", "cov-opt"])]),
"outdir": "worker-%d" % workerID,
"kcmd": " ".join(self.kleeCmd),
"debugcomm": "--debug-lb-communication" if self.debugcomm else "",
"debugcov": "--debug-coverable-instr" if self.debugcomm else "",
"maxtime": self.duration,
"coverable": "../%s" % os.path.basename(self.coverable),
"cmdline": cmdline,
"logdir": logdir,
"logfile": "%s/out-worker-%d.txt" % (logdir, workerID)
})
self._logMsg("Worker %d created on %s, port %d (lb. port %d) for target '%s'(%d)." % (workerID, host, port, lbPort, target, workerCount))
return proc
def _prepareLocalHost(self):
proc = runBashScript("""
if [ ! -f %(root)s/%(lb)s ]; then echo "Cannot find the load balancer executable: %(root)s/%(lb)s"; exit 1; fi
mkdir -p %(expdir)s/%(newdir)s
if [ -h %(expdir)s/last ]; then rm -f %(expdir)s/last; fi
[ ! -a %(expdir)s/last ] && ln -s %(expdir)s/%(newdir)s %(expdir)s/last""" % {
"root": self.localhost["root"],
"lb": LB_PATH,
"expdir": self.localhost["expdir"],
"newdir": self.uid
})
if proc.wait() != 0:
self._logMsg("Unable to initialize the local host ('%s'). Aborting..." % self.localhost["host"])
exit(1)
else:
self._logMsg("Local host initialized.")
def _runLB(self, port, target, workerCount, targetcounter):
logdir = "%s/%s" % (
self.localhost["expdir"],
self._getExperimentID(target, workerCount, targetcounter))
proc = runBashScript("""
mkdir -p %(logdir)s
%(root)s/%(lb)s -address %(address)s -port %(port)d %(debugcomm)s %(btout)s &>%(logfile)s""" % {
"logdir": logdir,
"root": self.localhost["root"],
"lb": LB_PATH,
"address": self.localhost["host"],
"port": port,
"debugcomm": "-debug-worker-communication" if self.debugcomm else "",
"btout": ("-balance-tout %d" % self.balancetout) if self.balancetout else "",
"logfile": "%s/out-lb.txt" % logdir
})
self._logMsg("Load balancer created for target '%s'(%d) on port %d." % (target, workerCount, port))
return proc
def _killAllRemote(self, host, signal="SIGINT", aggressive=False, freq=5):
proc = runBashScript("""
ssh -o StrictHostKeyChecking=no %(user)s@%(host)s 'bash -s' <<EOF
# The code below is run remotely
while true; do
DONE="true"
killall -%(signal)s %(worker)s %(klee)s %(lb)s && DONE="false"
%(mild)s && break
\\$DONE && break
echo "Tasks still running. Waiting more time..."
sleep %(freq)d
done
\nEOF""" % {
"user": self.hosts[host]["user"],
"host": host,
"signal": signal,
"worker": os.path.basename(WORKER_PATH),
"lb": os.path.basename(LB_PATH),
"klee": os.path.basename(KLEE_PATH),
"mild": "false" if aggressive else "true",
"freq": freq
})
proc.wait()
def _pollCoverage(self, host, testdirs, coveragedb, skip=5):
self._logMsg("Polling coverage for host %s..." % host)
proc = runBashScript("""
ssh %(user)s@%(host)s 'bash -s' <<EOF
cd %(expdir)s
# The code below is run remotely
for TESTDIR in %(testdirs)s; do
find ./\\$TESTDIR -name 'c9-coverage.txt' | while read LINE; do
echo \\$LINE
sed %(filter)s \\$LINE
done
done
\nEOF""" % {
"user": self.hosts[host]["user"],
"host": host,
"testdirs": " ".join(testdirs),
"expdir": self.hosts[host]["expdir"],
"filter": ("-n '1~%d p; $ p;'" % skip) if self.targetcov else "'$!N;$!D;'"
}, stdout=PIPE)
data,_ = proc.communicate()
targetData = None
for line in data.splitlines():
line = line.strip()
if not len(line):
continue
match = self.pathRe.match(line)
if match:
testdir, target, workercount, _, tgcount, workerID = match.groups()
workercount = int(workercount)
tgcount = int(tgcount) if tgcount else 1
workerID = int(workerID)
if isExperimentRejected(testdir, target, workercount, tgcount):
targetData = None
continue
targetData = coveragedb.setdefault(target, ToolData())
continue
if not targetData:
continue
try:
tokens = line.split()
timestamp = float(tokens[0])
globcov = None
total = 0
covered = 0
for t in tokens[1:]:
(k, v) = t.split("=")
(newcovAdd, totalAdd) = (int(x) for x in v.split("(")[0].split("/")[:2]) # 26/30(86.67)
if self.ffilter:
if k in self.ffilter:
total += totalAdd
covered += newcovAdd
elif k == "<global>":
total = totalAdd
covered = newcovAdd
break
globcov = 0 if total == 0 else 100. * covered / total
newcov = globcov
covdict = targetData.coverage.setdefault(workercount, {})
dataset = covdict.setdefault((testdir,tgcount), [])
dataset.append((timestamp, newcov))
except:
e_desc = traceback.format_exc()
self._logMsg("NOTE: Cannot process covdata '%s' on host '%s', target '%s'(%d), id %d, error: %s" % \
(line, host, target, workercount, workerID, e_desc))
def _pollStats(self, host, testdirs, statsdb, skip=5, wcfilter=None):
self._logMsg("Polling stats for host %s..." % host)
proc = runBashScript("""
ssh %(user)s@%(host)s 'bash -s' <<EOF
cd %(expdir)s
# The code below is run remotely
for TESTDIR in %(testdirs)s; do
find ./\\$TESTDIR -name 'c9-stats.txt' | while read LINE; do
echo \\$LINE
sed -n '1~%(skip)d p; $ p;' \\$LINE
echo
done
done
\nEOF""" % {
"user": self.hosts[host]["user"],
"host": host,
"testdirs": " ".join(testdirs),
"expdir": self.hosts[host]["expdir"],
"skip": skip
}, stdout=PIPE)
data,_ = proc.communicate()
entries = None
for line in data.splitlines():
line = line.strip()
if not len(line):
continue
match = self.pathRe.match(line)
if match:
testdir, target, workercount, _, tgcount, workerID = match.groups()
workercount = int(workercount)
tgcount = int(tgcount) if tgcount else 1
workerID = int(workerID)
if wcfilter and workercount != wcfilter:
entries = None
continue
if isExperimentRejected(testdir, target, workercount, tgcount):
entries = None
continue
tgdata = statsdb.setdefault(target, {})
expdata = tgdata.setdefault((testdir, tgcount), {})
wrkdata = expdata.setdefault(workercount, {})
entries = wrkdata.setdefault(workerID, [])
continue
if entries is None:
continue
tokpair = line.split(" ", 1)
timestamp = float(tokpair[0])
if len(tokpair) > 1:
stats = dict([(int(x[0]),int(x[1]))
for x in map(lambda token: token.split("="), tokpair[1].split())])
else:
stats = {}
entry = StatsEntry(timestamp, stats)
entries.append(entry)
return