def Run(self): logger = Logger() logger.LogStartMessage() generalParameters = GeneralParameters(self.cfgParser) reportParameters = ReportParameters(self.cfgParser) if self.parameters.doCheckout: repoParameters = RepoParameters(self.cfgParser) svn = SvnWorker(self.emailer, generalParameters, repoParameters) svn.CheckOut() if self.parameters.doBuild: tools = Tools(self.cfgParser) solutionBuilder = SolutionBuilder(self.emailer) solutionBuilder.BuildSln(generalParameters, tools) for experiment in self.experiments: logger.CoolLog("%s: starting %s" % (GetTimeStamp(), experiment.name)) launcher = ExperimentLauncher(experiment, self.storage, self.emailer) launcher.RunExperiment(generalParameters, reportParameters) self.storage.LogResults() self.storage.SendResults(self.emailer) self.comparator.CompareExperiments(reportParameters) logger.CoolLog("Finish")
def BuildSln(self, generalParameters, tools, mode="Rebuild"): logger = Logger() logger.CoolLog("Building solution...") slnPath = generalParameters.slnPath res = 0 #args = [tools.MSBuild, slnPath, "/t:" + mode, "/p:Configuration=Release"] args = "%s %s /t:%s /p:Configuration=Release" % (tools.MSBuild, slnPath, mode) logger.Log("Building solution using command: %s" % args) try: res = subprocess.call(args, shell=True) except WindowsError: error = ("Error: can not call %s" % Tools.MSBuild) ReportErrorAndExit(error, logger, self.emailer) except Exception: import traceback error = "Error: %s" % (traceback.format_exc()) ReportErrorAndExit(error, logger, self.emailer) if res != 0: error = "Build failed!" print(error) buildLog = generalParameters.buildLog if os.path.exists(buildLog): self.emailer.SendMessageAndExit(error, [buildLog]) else: self.emailer.SendMessageAndExit(error)
class ExperimentsComparator: logger = None storage = None masterExperiment = None masterResuts = {} #benchmark: pfst experimentsToCompare = { } #Group of experiments. Their results will be compared #experimentName: {benchmark: pfst} def __init__(self, storage): self.storage = storage self.masterExperiment = None self.masterResuts = {} self.experimentsToCompare = {} def AddExperimentToGroup(self, newExperiment): if self.masterExperiment is None: self.masterExperiment = newExperiment else: if newExperiment.benchmarks != self.masterExperiment.benchmarks: self.logger.Log("Error: list files are not equal!") exit(1) if newExperiment.metrics != self.masterExperiment.metrics: self.logger.Log("Error: metrics are not equal!") exit(1) self.experimentsToCompare[newExperiment.name] = {} def GetResultsForExperiment(self, experimentName): if experimentName in self.storage.experimentResults: return self.storage.experimentResults[experimentName].pfstTables self.logger.Log("Error: results for %s not found" % experimentName) return {} def GetExperimentsResults(self): #Masater experiment self.masterResuts = self.GetResultsForExperiment( self.masterExperiment.name) #Normal experiments for experimentName in self.experimentsToCompare.iterkeys(): self.experimentsToCompare[ experimentName] = self.GetResultsForExperiment(experimentName) @staticmethod def PrintLabelAndBlanks(cols, label, nMetrics): cols.extend(["", label]) #FIXME: bug, unused `col` variable for col in nMetrics: cols.append("") def PrintTableHeader(self, resultFileName): metrics = self.masterExperiment.metrics nMetrics = range(len(metrics)) # First line of the table header cols = [] self.PrintLabelAndBlanks(cols, "INIT", nMetrics) self.PrintLabelAndBlanks(cols, self.masterExperiment.name, nMetrics) for experimentName in self.experimentsToCompare.iterkeys(): self.PrintLabelAndBlanks(cols, experimentName, nMetrics) WriteStringToFile(cols, resultFileName) # Second line of the table header cols = ["benchmark"] #Init values """for row in nMetrics: cols.append(metrics[row])""" cols.extend(metrics) cols.append( "") #an empty column between INIT and experiments'es values #Target experiment's values ##for row in nMetrics: #TODO: check ## cols.append(metrics[row]) cols.extend(metrics) cols.append("") nNormalExperiments = len(self.experimentsToCompare) for experimentName in xrange(nNormalExperiments): for row in nMetrics: if metrics[row] != "Time": cols.append(metrics[row] + "%") else: cols.append(metrics[row]) cols.append("") #an empty column between experiments'es results WriteStringToFile(cols, resultFileName) def MakeResultTable(self, resultFileName): self.PrintTableHeader(resultFileName) benchmarks = list(self.masterResuts.keys()) for benchmark in benchmarks: newTableLine = self.CreateNewTableLine(benchmark) WriteStringToFile(newTableLine, resultFileName) def CreateNewTableLine(self, benchmark): metrics = self.masterExperiment.metrics nMetrics = range(len(metrics)) bestValues = [1000000 for i in nMetrics] bestValuesIdx = [0 for i in nMetrics] experimentsLine = [] # INIT metrics initialMetrics = [] initialMetricsLine = [] # reference metrics referenceMetrics = [] resultValues = self.masterResuts[benchmark] if resultValues != []: #If target experiment succeed on current benchmark for metricIdx in nMetrics: #Fill reference metrics finalStageIdx = len(self.masterExperiment.stages) - 1 value = resultValues[finalStageIdx][metricIdx] experimentsLine.append(str(value)) referenceMetrics.append(value) #Fill Initial metrics value = resultValues[0][metricIdx] initialMetricsLine.append(str(value)) initialMetrics.append(value) initialMetricsLine.append("") else: #Target experiment failed referenceMetrics = initialMetrics for metricIdx in nMetrics: experimentsLine.append("N/A") experimentsLine.append("") # normal experiments for experimentName, experimentResults in self.experimentsToCompare.iteritems( ): resultValues = experimentResults[benchmark] if resultValues == []: for metricIdx in nMetrics: experimentsLine.append("N/A") experimentsLine.append("") continue #if 'INITIAL' metrics haven't been printed yet if initialMetrics == []: #take them from the table of this experiment for metricIdx in nMetrics: value = resultValues[0][metricIdx] initialMetricsLine.append(str(value)) initialMetrics.append(value) initialMetricsLine.append("") #else compare 'INITIAL' metrics else: for metricIdx in nMetrics: cmpResult = CompareValues(initialMetrics[metricIdx], resultValues[0][metricIdx]) if cmpResult == NOT_EQUAL and metrics[metricIdx] != "Time": self.logger.Log("Error: not equal INIT metrics") finalStageIdx = -1 #Index of the last elemnt in the list for metricIdx in nMetrics: valueStr = resultValues[finalStageIdx][metricIdx] if metrics[metricIdx] != "Time": percent = 100 * valueStr / referenceMetrics[metricIdx] valueStr = "%.2f" % percent if percent < bestValues[metricIdx]: bestValues[metricIdx] = percent #remember the best result bestValuesIdx[metricIdx] = len( experimentsLine) #and its index experimentsLine.append(valueStr) experimentsLine.append("") #for idx in bestValuesIdx: # if (idx > 0): # experimentsLine[idx] = MarkResultAsBest(experimentsLine[idx]) if initialMetricsLine == []: for metricIdx in nMetrics: initialMetricsLine.append("N/A") initialMetricsLine.append("") newTableLine = [benchmark] newTableLine.extend(initialMetricsLine) newTableLine.extend(experimentsLine) return newTableLine def CompareExperiments(self, reportParameters): if len(self.experimentsToCompare) > 0: self.logger = Logger() self.logger.CoolLog("Comparing experiments") try: reportCreator = ReportCreator( "Comparing", "Comparing %s" % (GetTimeStamp()), reportParameters) logFolder = reportCreator.CreateLogFolder() cmpFileName = reportCreator.GetReportTableName() self.GetExperimentsResults() self.MakeResultTable(cmpFileName) except: import traceback self.logger.Log("exception: %s" % (traceback.format_exc())) self.logger.Log("Comparing finished")
class SvnWorker: logger = None emailer = None repoParameters = None generalParameters = None def __init__(self, emailer, generalParameters, repoParameters): self.emailer = emailer self.repoParameters = repoParameters self.generalParameters = generalParameters def DeleteSources(self): dir = self.generalParameters.checkoutPath self.logger.Log("Deleting previous version of itlPlace...") if os.path.exists(dir): RemoveDir(dir) self.logger.Log("Done") def doCheckOut(self, command): retcode = 1 for i in range(10): try: retcode = subprocess.call(command, shell=True) except Exception: import traceback error = "Error: can not call svn.exe: %s" % ( traceback.format_exc()) ReportErrorAndExit(error, self.logger, self.emailer) if retcode == 0: break if retcode != 0: error = "svn error: checkout failed!" ReportErrorAndExit(error, self.logger, self.emailer) return def FormCommand(self, rev=""): if rev != "": self.logger.Log("Checking out revision %s" % rev) rev = " -r %s" % rev else: self.logger.Log("Checking out HEAD revision") url = self.repoParameters.srcRepoPath to = self.generalParameters.checkoutPath #for testing... #url = 'http://svn.software.unn.ru/VLSI/CODE/trunk/TestingFramework' #to = '../../qqq' #command = "svn co --username belyakov --password 'njz%s %s %s"\ command = r'"C:\Program Files (x86)\CollabNet\Subversion Client\svn.exe" co%s %s %s' \ % (rev, url, to) return command def CheckOut(self): self.logger = Logger() self.logger.CoolLog("Delete sources and Checkout") self.DeleteSources() command = self.FormCommand() self.doCheckOut(command)