def _pollCoverage(self, host, testdirs, coveragedb, skip=5): self._logMsg("Polling coverage for host %s..." % host) proc = runBashScript(""" ssh %(user)s@%(host)s 'bash -s' <<EOF cd %(expdir)s # The code below is run remotely for TESTDIR in %(testdirs)s; do find ./\\$TESTDIR -name 'c9-coverage.txt' | while read LINE; do echo \\$LINE sed %(filter)s \\$LINE done done \nEOF""" % { "user": self.hosts[host]["user"], "host": host, "testdirs": " ".join(testdirs), "expdir": self.hosts[host]["expdir"], "filter": ("-n '1~%d p; $ p;'" % skip) if self.targetcov else "'$!N;$!D;'" }, stdout=PIPE) data,_ = proc.communicate() targetData = None for line in data.splitlines(): line = line.strip() if not len(line): continue match = self.pathRe.match(line) if match: testdir, target, workercount, _, tgcount, workerID = match.groups() workercount = int(workercount) tgcount = int(tgcount) if tgcount else 1 workerID = int(workerID) if isExperimentRejected(testdir, target, workercount, tgcount): targetData = None continue targetData = coveragedb.setdefault(target, ToolData()) continue if not targetData: continue try: tokens = line.split() timestamp = float(tokens[0]) globcov = None total = 0 covered = 0 for t in tokens[1:]: (k, v) = t.split("=") (newcovAdd, totalAdd) = (int(x) for x in v.split("(")[0].split("/")[:2]) # 26/30(86.67) if self.ffilter: if k in self.ffilter: total += totalAdd covered += newcovAdd elif k == "<global>": total = totalAdd covered = newcovAdd break globcov = 0 if total == 0 else 100. * covered / total newcov = globcov covdict = targetData.coverage.setdefault(workercount, {}) dataset = covdict.setdefault((testdir,tgcount), []) dataset.append((timestamp, newcov)) except: e_desc = traceback.format_exc() self._logMsg("NOTE: Cannot process covdata '%s' on host '%s', target '%s'(%d), id %d, error: %s" % \ (line, host, target, workercount, workerID, e_desc))
def _pollStats(self, host, testdirs, statsdb, skip=5, wcfilter=None): self._logMsg("Polling stats for host %s..." % host) proc = runBashScript(""" ssh %(user)s@%(host)s 'bash -s' <<EOF cd %(expdir)s # The code below is run remotely for TESTDIR in %(testdirs)s; do find ./\\$TESTDIR -name 'c9-stats.txt' | while read LINE; do echo \\$LINE sed -n '1~%(skip)d p; $ p;' \\$LINE echo done done \nEOF""" % { "user": self.hosts[host]["user"], "host": host, "testdirs": " ".join(testdirs), "expdir": self.hosts[host]["expdir"], "skip": skip }, stdout=PIPE) data,_ = proc.communicate() entries = None for line in data.splitlines(): line = line.strip() if not len(line): continue match = self.pathRe.match(line) if match: testdir, target, workercount, _, tgcount, workerID = match.groups() workercount = int(workercount) tgcount = int(tgcount) if tgcount else 1 workerID = int(workerID) if wcfilter and workercount != wcfilter: entries = None continue if isExperimentRejected(testdir, target, workercount, tgcount): entries = None continue tgdata = statsdb.setdefault(target, {}) expdata = tgdata.setdefault((testdir, tgcount), {}) wrkdata = expdata.setdefault(workercount, {}) entries = wrkdata.setdefault(workerID, []) continue if entries is None: continue tokpair = line.split(" ", 1) timestamp = float(tokpair[0]) if len(tokpair) > 1: stats = dict([(int(x[0]),int(x[1])) for x in map(lambda token: token.split("="), tokpair[1].split())]) else: stats = {} entry = StatsEntry(timestamp, stats) entries.append(entry) return