def createMergeScript(path, validations): if (len(validations) == 0): msg = "Cowardly refusing to merge nothing!" raise AllInOneError(msg) repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData": "", "CompareAlignments": "", "RunExtendedOfflineValidation": "" }) comparisonLists = { } # directory of lists containing the validations that are comparable resultPlotFile = "" # string of a file name for createExtendedValidationScript for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s" % (validation.__class__.__name__, referenceName) validationName = validationName.split( ".%s" % GenericValidation.defaultReferenceName)[0] if validationName in comparisonLists: comparisonLists[validationName].append(validation) else: comparisonLists[validationName] = [validation] if validationName == "OfflineValidation": resultPlotFile = validationName if "OfflineValidation" in comparisonLists: repMap["extendeValScriptPath"] = \ os.path.join(path, "TkAlExtendedOfflineValidation.C") createExtendedValidationScript(comparisonLists["OfflineValidation"], repMap["extendeValScriptPath"], resultPlotFile) repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) repMap["CompareAlignments"] = "#run comparisons" for validationId in comparisonLists: compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] repMap.update({ "validationId": validationId, "compareStrings": " , ".join(compareStrings) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open(filePath, "w") theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap)) theFile.close() os.chmod(filePath, 0755) return filePath
def createMergeScript( path, validations ): if(len(validations) == 0): msg = "Cowardly refusing to merge nothing!" raise AllInOneError(msg) repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData":"", "CompareAlignments":"", "RunExtendedOfflineValidation":"", "RunTrackSplitPlot":"" }) comparisonLists = {} # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s"%(validation.__class__.__name__, referenceName) validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0] if validationName in comparisonLists: comparisonLists[ validationName ].append( validation ) else: comparisonLists[ validationName ] = [ validation ] if "OfflineValidation" in comparisonLists: repMap["extendeValScriptPath"] = \ os.path.join(path, "TkAlExtendedOfflineValidation.C") createExtendedValidationScript(comparisonLists["OfflineValidation"], repMap["extendeValScriptPath"], "OfflineValidation") repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) if "TrackSplittingValidation" in comparisonLists: repMap["trackSplitPlotScriptPath"] = \ os.path.join(path, "TkAlTrackSplitPlot.C") createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"], repMap["trackSplitPlotScriptPath"] ) repMap["RunTrackSplitPlot"] = \ replaceByMap(configTemplates.trackSplitPlotExecution, repMap) repMap["CompareAlignments"] = "#run comparisons" for validationId in comparisonLists: compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ] repMap.update({"validationId": validationId, "compareStrings": " , ".join(compareStrings), "compareStringsPlain": " ".join(compareStringsPlain) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open( filePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) theFile.close() os.chmod(filePath,0755) return filePath
def createParallelMergeScript( path, validations ): if( len(validations) == 0 ): raise AllInOneError, "cowardly refusing to merge nothing!" repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData":"", "CompareAlignments":"", "RunExtendedOfflineValidation":"" }) comparisonLists = {} # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s"%(validation.__class__.__name__, referenceName) validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0] if validationName in comparisonLists: comparisonLists[ validationName ].append( validation ) else: comparisonLists[ validationName ] = [ validation ] if "OfflineValidationParallel" in comparisonLists: repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C") createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] ) repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C") createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"], repMap["mergeOfflineParJobsScriptPath"] ) repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) # needed to copy the right merged file back to eos repMap["resultFile"] = comparisonLists["OfflineValidationParallel"][0].getRepMap()["resultFile"] repMap["outputFile"] = comparisonLists["OfflineValidationParallel"][0].getRepMap()["outputFile"] # DownloadData is the section which merges output files from parallel jobs # it uses the file TkAlOfflineJobsMerge.C repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap) repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap ) repMap["CompareAlignments"] = "#run comparisons" for validationId in comparisonLists: compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] repMap.update({"validationId": validationId, "compareStrings": " , ".join(compareStrings) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open( filePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) theFile.close() os.chmod(filePath,0755) return filePath
def submitjob(jobtext, jobname=None, jobtime=None, queue=None, interactive=False, waitids=[], waitsuccessids=[], outputfile=None, errorfile=None, docd=False, morerepmap=None, email=True): if not email: raise RuntimeError("Not sure how to turn off email for lxplus") if outputfile is not None: outputfile = outputfile.format(jobid="%J") if errorfile is not None: errorfile = errorfile.format(jobid="%J") if interactive: queue = "cmsinter" if queue is None: if jobtime is None: raise ValueError("Have to provide either queue or jobtime (or set interactive=True)!") #http://stackoverflow.com/a/4628148/5228524 match = re.match(r'((?P<days>\d+?)-)?((?P<hours>\d+?):)?((?P<minutes>\d+?):)?((?P<seconds>\d+?))?', jobtime) if not match: raise ValueError("Bad time string {}".format(jobtime)) kwargs = {key: int(value) for key, value in match.groupdict().iteritems() if value is not None} delta = timedelta(**kwargs) if not delta: raise ValueError("Job takes no time! {}".format(jobtime)) if delta <= timedelta(minutes=8): queue = "8nm" elif delta <= timedelta(hours=1): queue = "1nh" elif delta <= timedelta(hours=8): queue = "8nh" elif delta <= timedelta(days=1): queue = "1nd" elif delta <= timedelta(days=2): queue = "2nd" elif delta <= timedelta(weeks=1): queue = "1nw" elif delta <= timedelta(weeks=2): queue = "2nw" else: raise ValueError("time {} is more than 2 weeks!".format(jobtime)) run = """ echo .oO[jobtext]Oo. | bsub .oO[options]Oo. """ optionsdict = { "-q": queue, "-J": jobname, "-o": outputfile, "-e": errorfile, "-w": " && ".join(["ended({:d})".format(id) for id in waitids]+["done({:d})".format(id) for id in waitsuccessids]) } repmap = { "jobtext": quote("cd .oO[CMSSW_BASE]Oo. && eval $(scram ru -sh) && .oO[docd]Oo. && "+jobtext), "CMSSW_BASE": os.environ["CMSSW_BASE"], "pwd": os.getcwd(), "options": " ".join("{} {}".format(key, quote(value)) for key, value in optionsdict.iteritems() if value), "docd": "cd .oO[pwd]Oo." if docd else "cd -" } if morerepmap: assert not (set(repmap) & set(morerepmap)) repmap.update(morerepmap) if interactive: repmap["options"] += " -I" subprocess.check_call(replaceByMap(run, repmap), shell=True) else: bsubout = subprocess.check_output(replaceByMap(run, repmap), shell=True) jobid = int(bsubout.split("<")[1].split(">")[0]) print bsubout, return jobid
def createOfflineJobsMergeScript(offlineValidationList, outFilePath): repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) ) theFile.close()
def createOfflineParJobsMergeScript(offlineValidationList, outFilePath): repMap = offlineValidationList[0].getRepMap( ) # bit ugly since some special features are filled theFile = open(outFilePath, "w") theFile.write( replaceByMap(configTemplates.mergeOfflineParJobsTemplate, repMap)) theFile.close()
def createExtendedValidationScript(offlineValidationList, outFilePath): repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in offlineValidationList: repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] ) theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) ) theFile.close()
def createMergeZmumuPlotsScript(zMuMuValidationList, outFilePath): repMap = zMuMuValidationList[0].getRepMap() # bit ugly since some special features are filled repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE'] repMap[ "mergeZmumuPlotsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in zMuMuValidationList: repMap[ "mergeZmumuPlotsInstantiation" ] = validation.appendToExtendedValidation( repMap[ "mergeZmumuPlotsInstantiation" ] ) theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeZmumuPlotsTemplate ,repMap ) ) theFile.close()
def createOfflineJobsMergeScript(offlineValidationList, outFilePath): repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in offlineValidationList: repMap[ "mergeOfflinParJobsInstantiation" ] = validation.appendToMergeParJobs( repMap[ "mergeOfflinParJobsInstantiation" ] ) # validationsSoFar = 'PlotAlignmentValidation p("%(resultFile)s", "%(name)s", %(color)s, %(style)s);\n'%repMap theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) ) theFile.close()
def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath): repMap = trackSplittingValidationList[0].getRepMap() # bit ugly since some special features are filled repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in trackSplittingValidationList: repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] ) theFile = open( outFilePath, "w" ) # theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) ) theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) ) theFile.close()
def createMergeZmumuPlotsScript(zMuMuValidationList, outFilePath): config = zMuMuValidationList[0].config repMap = PlottingOptions(config, "zmumu") repMap[ "mergeZmumuPlotsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in zMuMuValidationList: repMap[ "mergeZmumuPlotsInstantiation" ] = validation.appendToExtendedValidation( repMap[ "mergeZmumuPlotsInstantiation" ] ) theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeZmumuPlotsTemplate ,repMap ) ) theFile.close()
def createPrimaryVertexPlotScript(PrimaryVertexValidationList, outFilePath): repMap = PrimaryVertexValidationList[0].getRepMap() # bit ugly since some special features are filled repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE'] repMap[ "PrimaryVertexPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in PrimaryVertexValidationList: repMap[ "PrimaryVertexPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "PrimaryVertexPlotInstantiation" ] ) theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.PrimaryVertexPlotTemplate ,repMap ) ) theFile.close()
def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath): config = trackSplittingValidationList[0].config repMap = PlottingOptions(config, "split") repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in trackSplittingValidationList: repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] ) theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) ) theFile.close()
def createExtendedValidationScript(offlineValidationList, outFilePath): repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in offlineValidationList: repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] ) theFile = open( outFilePath, "w" ) # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) ) theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) ) theFile.close()
def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile): config = offlineValidationList[0].config repMap = PlottingOptions(config, "offline") repMap[ "resultPlotFile" ] = resultPlotFile repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back for validation in offlineValidationList: repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] ) theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) ) theFile.close()
def createOfflineJobsMergeScript(offlineValidationList, outFilePath): repMap = offlineValidationList[0].getRepMap( ) # bit ugly since some special features are filled repMap[ "mergeOfflinParJobsInstantiation"] = "" #give it a "" at first in order to get the initialisation back for validation in offlineValidationList: repMap[ "mergeOfflinParJobsInstantiation"] = validation.appendToMergeParJobs( repMap["mergeOfflinParJobsInstantiation"]) # validationsSoFar = 'PlotAlignmentValidation p("%(resultFile)s", "%(name)s", %(color)s, %(style)s);\n'%repMap theFile = open(outFilePath, "w") theFile.write( replaceByMap(configTemplates.mergeOfflineParJobsTemplate, repMap)) theFile.close()
def createOfflineParJobsMergeScript(offlineValidationList, outFilePath): repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled theFile = open( outFilePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) ) theFile.close()
def createMergeScript(path, validations, options): if (len(validations) == 0): raise AllInOneError("Cowardly refusing to merge nothing!") config = validations[0].config repMap = config.getGeneral() repMap.update({ "DownloadData": "", "CompareAlignments": "", "RunValidationPlots": "", "CMSSW_BASE": os.environ["CMSSW_BASE"], "SCRAM_ARCH": os.environ["SCRAM_ARCH"], "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"], }) comparisonLists = { } # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationtype = type(validation) if issubclass(validationtype, PreexistingValidation): #find the actual validationtype for parentclass in validationtype.mro(): if not issubclass(parentclass, PreexistingValidation): validationtype = parentclass break key = (validationtype, referenceName) if key in comparisonLists: comparisonLists[key].append(validation) else: comparisonLists[key] = [validation] # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["doMerge"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = ( "if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n") repMap["beforeMerge"] = "" repMap["mergeParallelFilePrefixes"] = "" repMap["createResultsDirectory"] = "" anythingToMerge = [] #prepare dictionary containing handle objects for parallel merge batch jobs if options.mergeOfflineParallel: parallelMergeObjects = {} for (validationType, referencename), validations in six.iteritems(comparisonLists): for validation in validations: #parallel merging if (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)): continue if options.mergeOfflineParallel and validationType.valType == 'offline' and validation.jobmode.split( ",")[0] == "lxBatch": repMapTemp = repMap.copy() if validationType not in anythingToMerge: anythingToMerge += [validationType] #create init script fileName = "TkAlMergeInit" filePath = os.path.join(path, fileName + ".sh") theFile = open(filePath, "w") repMapTemp["createResultsDirectory"] = "#!/bin/bash" repMapTemp["createResultsDirectory"] += replaceByMap( configTemplates.createResultsDirectoryTemplate, repMapTemp) theFile.write( replaceByMap( configTemplates.createResultsDirectoryTemplate, repMapTemp)) theFile.close() os.chmod(filePath, 0o755) #create handle parallelMergeObjects["init"] = ParallelMergeJob( fileName, filePath, []) #clear 'create result directory' code repMapTemp["createResultsDirectory"] = "" #edit repMapTmp as necessary: #fill contents of mergeParallelResults repMapTemp["beforeMerge"] += validationType.doInitMerge() repMapTemp[ "doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs with alignment %s"\n\n' % ( validationType.valType, validation.alignmentToValidate.name) repMapTemp["doMerge"] += validation.doMerge() for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/eos/cms/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMapTemp["rmUnmerged"] += " rm " + longName + "\n" repMapTemp["rmUnmerged"] += ( "else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") #fill mergeParallelResults area of mergeTemplate repMapTemp["DownloadData"] = replaceByMap( configTemplates.mergeParallelResults, repMapTemp) #fill runValidationPlots area of mergeTemplate repMapTemp["RunValidationPlots"] = validationType.doRunPlots( validations) #create script file fileName = "TkAlMergeOfflineValidation" + validation.name + validation.alignmentToValidate.name filePath = os.path.join(path, fileName + ".sh") theFile = open(filePath, "w") theFile.write( replaceByMap(configTemplates.mergeParallelOfflineTemplate, repMapTemp)) theFile.close() os.chmod(filePath, 0o755) #create handle object if "parallel" in parallelMergeObjects: parallelMergeObjects["parallel"].append( ParallelMergeJob(fileName, filePath, [])) else: parallelMergeObjects["parallel"] = [ ParallelMergeJob(fileName, filePath, []) ] continue else: if validationType not in anythingToMerge: anythingToMerge += [validationType] repMap[ "doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType repMap["beforeMerge"] += validationType.doInitMerge() repMap["doMerge"] += validation.doMerge() for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/eos/cms/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " rm " + longName + "\n" repMap["rmUnmerged"] += ( "else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") if anythingToMerge: repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap) else: repMap["DownloadData"] = "" repMap["RunValidationPlots"] = "" for (validationType, referencename), validations in six.iteritems(comparisonLists): if issubclass(validationType, ValidationWithPlots): repMap["RunValidationPlots"] += validationType.doRunPlots( validations) repMap["CompareAlignments"] = "#run comparisons" for (validationType, referencename), validations in six.iteritems(comparisonLists): if issubclass(validationType, ValidationWithComparison): repMap["CompareAlignments"] += validationType.doComparison( validations) #if user wants to merge parallely and if there are valid parallel scripts, create handle for plotting job and set merge script name accordingly if options.mergeOfflineParallel and parallelMergeObjects != {}: parallelMergeObjects["continue"] = ParallelMergeJob( "TkAlMergeFinal", os.path.join(path, "TkAlMergeFinal.sh"), []) filePath = os.path.join(path, "TkAlMergeFinal.sh") #if not merging parallel, add code to create results directory and set merge script name accordingly else: repMap["createResultsDirectory"] = replaceByMap( configTemplates.createResultsDirectoryTemplate, repMap) filePath = os.path.join(path, "TkAlMerge.sh") #filePath = os.path.join(path, "TkAlMerge.sh") theFile = open(filePath, "w") theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap)) theFile.close() os.chmod(filePath, 0o755) if options.mergeOfflineParallel: return { 'TkAlMerge.sh': filePath, 'parallelMergeObjects': parallelMergeObjects } else: return filePath
def submitjob(jobtext, jobname=None, jobtime=None, queue=None, interactive=False, waitids=[], outputfile=None, errorfile=None, docd=False, morerepmap=None, email=False, memory="3000M"): if queue is None: queue = "shared" if outputfile is not None: outputfile = outputfile.format(jobid="%j") if errorfile is not None: errorfile = errorfile.format(jobid="%j") jobtemplate = textwrap.dedent(""" #!/bin/bash . /work-zfs/lhc/cms/cmsset_default.sh && cd .oO[CMSSW_BASE]Oo. && eval $(scram ru -sh) && .oO[docd]Oo. && echo "SLURM job running in: " `pwd` && #commands .oO[jobtext]Oo. """).strip() repmap = { "jobtext": jobtext, "CMSSW_BASE": os.environ["CMSSW_BASE"], "pwd": os.getcwd(), "jobname": jobname, "jobtime": jobtime, "queue": queue, "waitids": ":".join("{:d}".format(id) for id in waitids), "docd": "cd .oO[pwd]Oo." if docd else "cd $(mktemp -d)", } if morerepmap: assert not (set(repmap) & set(morerepmap)) repmap.update(morerepmap) options = { "--job-name": ".oO[jobname]Oo.", "--time": ".oO[jobtime]Oo.", "--nodes": "1", "--ntasks-per-node": "1", "--partition": ".oO[queue]Oo.", "--mem": memory, "--output": outputfile, "--error": errorfile, } if waitids: options["--dependency"] = "afterany:.oO[waitids]Oo." if email: options["--mail-user"] = email options["--mail-type"] = "end" options = {replaceByMap(k, repmap): replaceByMap(v, repmap) for k, v in options.iteritems() if v is not None} options = ["{}={}".format(k, quote(v)) for k, v in options.iteritems()] if interactive: subprocess.check_call( ["srun"] + options + ["bash", "-c", replaceByMap(".oO[jobtext]Oo.", repmap)] ) else: with tempfile.NamedTemporaryFile(bufsize=0) as f: f.write(replaceByMap(jobtemplate, repmap)) os.chmod(f.name, os.stat(f.name).st_mode | stat.S_IEXEC) sbatchout = subprocess.check_output(["sbatch"] + options + [f.name]) jobid = int(sbatchout.split()[-1]) print sbatchout, return jobid
def runcombine(analysis, foldername, **kwargs): usechannels = channels expectvalues = [0.0] plotname = "limit" legendposition = (.2, .7, .6, .9) CLtextposition = "left" productions = config.productionsforcombine usesystematics = True usebkg = True runobs = True defaultscanrange = scanrange = (-1.0, 1.0) defaultnpoints = npoints = 101 defaultscenario = scenario = 1 scalemuvmuftogether = False for kw, kwarg in kwargs.iteritems(): if kw == "channels": usechannels = [Channel(c) for c in kwarg.split(",")] elif kw == "expectvalues": try: expectvalues = [fai if fai=="minimum" else float(fai) for fai in kwarg.split(",")] except ValueError: if kwarg == "": expectvalues = [] else: raise ValueError("expectvalues has to contain floats separated by commas!") elif kw == "plotname": plotname = kwarg elif kw == "CLtextposition": CLtextposition = kwarg elif kw == "legendposition": try: legendposition = [float(a) for a in kwarg.split(",")] if len(legendposition) != 4: raise ValueError except ValueError: raise ValueError("legendposition has to contain 4 floats separated by commas!") elif kw == "productions": productions = [Production(p) for p in kwarg.split(",")] elif kw == "usesystematics": usesystematics = bool(int(kwarg)) elif kw == "usebkg": usebkg = bool(int(kwarg)) elif kw == "scanrange": try: scanrange = tuple(float(a) for a in kwarg.split(",")) if len(scanrange) != 2: raise ValueError except ValueError: raise ValueError("scanrange has to contain 2 floats separated by a comma!") elif kw == "npoints": npoints = int(kwarg) elif kw == "scenario": scenario = int(kwarg) elif kw == "scalemuvmuftogether": scalemuvmuftogether = bool(int(kwarg)) elif kw == "runobs": runobs = bool(int(kwarg)) else: raise TypeError("Unknown kwarg: {}".format(kw)) if not config.unblindscans: luminosity = float(Luminosity("forexpectedscan")) else: luminosity = None years = [p.year for p in productions] if len(set(years)) != len(years): raise ValueError("Some of your productions are from the same year!") workspacefileappend = "" if scalemuvmuftogether: workspacefileappend += "_scalemuvmuftogether" print workspacefileappend moreappend = ".oO[workspacefileappend]Oo." if not usesystematics: moreappend += "_nosystematics" if not (npoints == defaultnpoints and scanrange == defaultscanrange): moreappend += "_{},{},{}".format(npoints, *scanrange) analysis = Analysis(analysis) foldername = "{}_{}".format(analysis, foldername) if scenario != defaultscenario: foldername += "_scenario{}".format(scenario) repmap = { "foldername": pipes.quote(foldername), "analysis": str(analysis), "cardstocombine": " ".join("hzz4l_{}S_{}.txt".format(channel, production.year) for channel, production in product(usechannels, productions)), "workspacefile": "floatMu.oO[workspacefileappend]Oo..root", "filename": "higgsCombine_.oO[append]Oo..oO[moreappend]Oo..MultiDimFit.mH125.root", "expectedappend": "exp_.oO[expectfai]Oo.", "observedappend": "obs", "workspacefileappend": workspacefileappend, "usesystematics": str(int(usesystematics)), "moreappend": moreappend, "npoints": str(npoints), "scanrange": "{},{}".format(*scanrange), "SM_XS_VV": str(analysis.SM_XS_VV), "BSM_XS_VV": str(analysis.BSM_XS_VV), "int_XS_VV": str(analysis.int_XS_VV), "morePO": "--PO scalemuvmuftogether" if scalemuvmuftogether else "", } with filemanager.cd(os.path.join(config.repositorydir, "CMSSW_8_1_0/src/HiggsAnalysis/HZZ4l_Combination/CreateDatacards")): for production in productions: production = Production(production) if not all(os.path.exists("cards_{}/HCG/125/{}/hzz4l_{}S_{}.input.root".format(foldername, production.year, channel, production.year)) for channel in usechannels): makeworkspacesmap = repmap.copy() makeworkspacesmap["production"] = str(production) subprocess.check_call(replaceByMap(makeworkspacestemplate, makeworkspacesmap), shell=True) with open("cards_{}/.gitignore".format(foldername), "w") as f: f.write("*") with filemanager.cd("cards_{}/HCG/125/{}".format(foldername, production.year)): #replace rates for channel, production in product(channels, productions): if channel in usechannels: with open("hzz4l_{}S_{}.txt".format(channel, production.year)) as f: contents = f.read() if "\n#rate" in contents: continue #already did this for line in contents.split("\n"): if line.startswith("rate"): if config.unblindscans: rates = getrates(channel, "fordata", production, usebkg=usebkg) else: rates = getrates(channel, "forexpectedscan", production, usebkg=usebkg) contents = contents.replace(line, "#"+line+"\n"+rates) break with open("hzz4l_{}S_{}.txt".format(channel, production.year), "w") as f: f.write(contents) else: if os.path.exists("hzz4l_{}S_{}.txt".format(channel, production.year)): os.remove("hzz4l_{}S_{}.txt".format(channel, production.year)) os.remove("hzz4l_{}S_{}.input.root".format(channel, production.year)) if not os.path.exists(replaceByMap(".oO[workspacefile]Oo.", repmap)): for channel, production in product(usechannels, productions): replacesystematics(channel, production, scenario=scenario, luminosity=luminosity) subprocess.check_call(replaceByMap(createworkspacetemplate, repmap), shell=True) if config.unblindscans and runobs: repmap_obs = repmap.copy() repmap_obs["expectfai"] = "0.0" #starting point repmap_obs["append"] = ".oO[observedappend]Oo." if not os.path.exists(replaceByMap(".oO[filename]Oo.", repmap_obs)): subprocess.check_call(replaceByMap(observationcombinetemplate, repmap_obs), shell=True) f = ROOT.TFile(replaceByMap(".oO[filename]Oo.", repmap_obs)) minimum = (float("nan"), float("inf")) for entry in f.limit: if f.limit.deltaNLL < minimum[1]: minimum = (f.limit.CMS_zz4l_fai1, f.limit.deltaNLL) minimum = minimum[0] del f for expectfai in expectvalues: repmap_exp = repmap.copy() if expectfai == "minimum": expectfai = minimum repmap_exp["expectfai"] = str(expectfai) repmap_exp["append"] = ".oO[expectedappend]Oo." if not os.path.exists(replaceByMap(".oO[filename]Oo.", repmap_exp)): subprocess.check_call(replaceByMap(runcombinetemplate, repmap_exp), shell=True) saveasdir = os.path.join(config.plotsbasedir, "limits", foldername) try: os.makedirs(saveasdir) except OSError: pass plotscans = [] if config.unblindscans and runobs: plotscans.append("obs") for expectfai in expectvalues: if expectfai == "minimum": expectfai = minimum plotscans.append(expectfai) for ext in "png eps root pdf".split(): plotname = plotname.replace("."+ext, "") plotname += moreappend plotname = replaceByMap(plotname, repmap) plotlimits(os.path.join(saveasdir, plotname), analysis, *plotscans, productions=productions, legendposition=legendposition, CLtextposition=CLtextposition, moreappend=replaceByMap(moreappend, repmap)) with open(os.path.join(saveasdir, plotname+".txt"), "w") as f: f.write(" ".join(["python"]+sys.argv)) f.write("\n\n\n") f.write("python limits.py ") for arg in sys.argv[1:]: if "=" in arg: continue f.write(arg+" ") f.write("plotname="+plotname+" ") f.write("\n")
def createParallelMergeScript(path, validations): if (len(validations) == 0): raise AllInOneError, "cowardly refusing to merge nothing!" repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData": "", "CompareAlignments": "", "RunExtendedOfflineValidation": "", "RunTrackSplitPlot": "" }) comparisonLists = { } # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s" % (validation.__class__.__name__, referenceName) validationName = validationName.split( ".%s" % GenericValidation.defaultReferenceName)[0] if validationName in comparisonLists: comparisonLists[validationName].append(validation) else: comparisonLists[validationName] = [validation] if "OfflineValidationParallel" in comparisonLists: repMap["extendeValScriptPath"] = os.path.join( path, "TkAlExtendedOfflineValidation.C") repMap["mergeOfflineParJobsScriptPath"] = os.path.join( path, "TkAlOfflineJobsMerge.C") createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"], repMap["mergeOfflineParJobsScriptPath"]) # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["haddLoop"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = ( "if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n") for validation in comparisonLists["OfflineValidationParallel"]: repMap["haddLoop"] = validation.appendToMergeParJobs( repMap["haddLoop"]) repMap["haddLoop"] += "tmpMergeRetCode=${?}\n" repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then " "cmsStage -f " + validation.getRepMap()["outputFile"] + " " + validation.getRepMap()["resultFile"] + "; fi\n") repMap["haddLoop"] += ( "if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then " "mergeRetCode=${tmpMergeRetCode}; fi\n") for f in validation.outputFiles: longName = os.path.join("/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " cmsRm " + longName + "\n" repMap["rmUnmerged"] += ( "else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) # DownloadData is the section which merges output files from parallel jobs # it uses the file TkAlOfflineJobsMerge.C repMap["DownloadData"] += replaceByMap( "rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap) repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap) # For the rest of the script, OfflineValidations and # OfflineParallelValidations are comparable: if "OfflineValidation" in comparisonLists: comparisonLists["OfflineValidationParallel"].extend( comparisonLists["OfflineValidation"]) del comparisonLists["OfflineValidation"] createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"], "OfflineValidationParallel") repMap["CompareAlignments"] = "#run comparisons" for validationId in comparisonLists: compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ] repMap.update({ "validationId": validationId, "compareStrings": " , ".join(compareStrings), "compareStringsPlain": " ".join(compareStringsPlain) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open(filePath, "w") theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap)) theFile.close() os.chmod(filePath, 0755) return filePath
def createMergeScript( path, validations ): if(len(validations) == 0): raise AllInOneError("Cowardly refusing to merge nothing!") config = validations[0].config repMap = config.getGeneral() repMap.update({ "DownloadData":"", "CompareAlignments":"", "RunValidationPlots":"", "CMSSW_BASE": os.environ["CMSSW_BASE"], "SCRAM_ARCH": os.environ["SCRAM_ARCH"], "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"], }) comparisonLists = {} # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationtype = type(validation) if isinstance(validationtype, PreexistingValidation): #find the actual validationtype for parentclass in validationtype.mro(): if not issubclass(parentclass, PreexistingValidation): validationtype = parentclass break key = (validationtype, referenceName) if key in comparisonLists: comparisonLists[key].append(validation) else: comparisonLists[key] = [validation] # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["doMerge"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n") repMap["beforeMerge"] = "" repMap["mergeParallelFilePrefixes"] = "" anythingToMerge = [] for (validationType, referencename), validations in comparisonLists.iteritems(): for validation in validations: if (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)): continue if validationType not in anythingToMerge: anythingToMerge += [validationType] repMap["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType repMap["beforeMerge"] += validationType.doInitMerge() repMap["doMerge"] += validation.doMerge() for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " $eos rm "+longName+"\n" repMap["rmUnmerged"] += ("else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") if anythingToMerge: repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap ) else: repMap["DownloadData"] = "" repMap["RunValidationPlots"] = "" for (validationType, referencename), validations in comparisonLists.iteritems(): if issubclass(validationType, ValidationWithPlots): repMap["RunValidationPlots"] += validationType.doRunPlots(validations) repMap["CompareAlignments"] = "#run comparisons" for (validationType, referencename), validations in comparisonLists.iteritems(): if issubclass(validationType, ValidationWithComparison): repMap["CompareAlignments"] += validationType.doComparison(validations) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open( filePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) theFile.close() os.chmod(filePath,0o755) return filePath
def createParallelMergeScript(path, validations): if (len(validations) == 0): raise AllInOneError, "cowardly refusing to merge nothing!" repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData": "", "CompareAlignments": "", "RunExtendedOfflineValidation": "" }) comparisonLists = { } # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s" % (validation.__class__.__name__, referenceName) validationName = validationName.split( ".%s" % GenericValidation.defaultReferenceName)[0] if validationName in comparisonLists: comparisonLists[validationName].append(validation) else: comparisonLists[validationName] = [validation] if "OfflineValidationParallel" in comparisonLists: repMap["extendeValScriptPath"] = os.path.join( path, "TkAlExtendedOfflineValidation.C") createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"]) repMap["mergeOfflineParJobsScriptPath"] = os.path.join( path, "TkAlOfflineJobsMerge.C") createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"], repMap["mergeOfflineParJobsScriptPath"]) repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) # needed to copy the right merged file back to eos repMap["resultFile"] = comparisonLists["OfflineValidationParallel"][ 0].getRepMap()["resultFile"] repMap["outputFile"] = comparisonLists["OfflineValidationParallel"][ 0].getRepMap()["outputFile"] # DownloadData is the section which merges output files from parallel jobs # it uses the file TkAlOfflineJobsMerge.C repMap["DownloadData"] += replaceByMap( "rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap) repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap) repMap["CompareAlignments"] = "#run comparisons" for validationId in comparisonLists: compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] repMap.update({ "validationId": validationId, "compareStrings": " , ".join(compareStrings) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open(filePath, "w") theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap)) theFile.close() os.chmod(filePath, 0755) return filePath
def createMergeScript( path, validations, options ): if(len(validations) == 0): raise AllInOneError("Cowardly refusing to merge nothing!") config = validations[0].config repMap = config.getGeneral() repMap.update({ "DownloadData":"", "CompareAlignments":"", "RunValidationPlots":"", "CMSSW_BASE": os.environ["CMSSW_BASE"], "SCRAM_ARCH": os.environ["SCRAM_ARCH"], "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"], }) comparisonLists = {} # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationtype = type(validation) if issubclass(validationtype, PreexistingValidation): #find the actual validationtype for parentclass in validationtype.mro(): if not issubclass(parentclass, PreexistingValidation): validationtype = parentclass break key = (validationtype, referenceName) if key in comparisonLists: comparisonLists[key].append(validation) else: comparisonLists[key] = [validation] # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["doMerge"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n") repMap["beforeMerge"] = "" repMap["mergeParallelFilePrefixes"] = "" repMap["createResultsDirectory"]="" anythingToMerge = [] #prepare dictionary containing handle objects for parallel merge batch jobs if options.mergeOfflineParallel: parallelMergeObjects={} for (validationType, referencename), validations in six.iteritems(comparisonLists): for validation in validations: #parallel merging if (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)): continue if options.mergeOfflineParallel and validationType.valType=='offline' and validation.jobmode.split(",")[0]=="lxBatch": repMapTemp=repMap.copy() if validationType not in anythingToMerge: anythingToMerge += [validationType] #create init script fileName="TkAlMergeInit" filePath = os.path.join(path, fileName+".sh") theFile = open( filePath, "w" ) repMapTemp["createResultsDirectory"]="#!/bin/bash" repMapTemp["createResultsDirectory"]+=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMapTemp) theFile.write( replaceByMap( configTemplates.createResultsDirectoryTemplate, repMapTemp ) ) theFile.close() os.chmod(filePath,0o755) #create handle parallelMergeObjects["init"]=ParallelMergeJob(fileName, filePath,[]) #clear 'create result directory' code repMapTemp["createResultsDirectory"]="" #edit repMapTmp as necessary: #fill contents of mergeParallelResults repMapTemp["beforeMerge"] += validationType.doInitMerge() repMapTemp["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs with alignment %s"\n\n' % (validationType.valType,validation.alignmentToValidate.name) repMapTemp["doMerge"] += validation.doMerge() for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/eos/cms/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMapTemp["rmUnmerged"] += " rm "+longName+"\n" repMapTemp["rmUnmerged"] += ("else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") #fill mergeParallelResults area of mergeTemplate repMapTemp["DownloadData"] = replaceByMap( configTemplates.mergeParallelResults, repMapTemp ) #fill runValidationPlots area of mergeTemplate repMapTemp["RunValidationPlots"] = validationType.doRunPlots(validations) #create script file fileName="TkAlMerge"+validation.alignmentToValidate.name filePath = os.path.join(path, fileName+".sh") theFile = open( filePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeParallelOfflineTemplate, repMapTemp ) ) theFile.close() os.chmod(filePath,0o755) #create handle object if "parallel" in parallelMergeObjects: parallelMergeObjects["parallel"].append(ParallelMergeJob(fileName, filePath,[])) else: parallelMergeObjects["parallel"]=[ParallelMergeJob(fileName, filePath,[])] continue else: if validationType not in anythingToMerge: anythingToMerge += [validationType] repMap["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType repMap["beforeMerge"] += validationType.doInitMerge() repMap["doMerge"] += validation.doMerge() for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/eos/cms/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " rm "+longName+"\n" repMap["rmUnmerged"] += ("else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") if anythingToMerge: repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap ) else: repMap["DownloadData"] = "" repMap["RunValidationPlots"] = "" for (validationType, referencename), validations in six.iteritems(comparisonLists): if issubclass(validationType, ValidationWithPlots): repMap["RunValidationPlots"] += validationType.doRunPlots(validations) repMap["CompareAlignments"] = "#run comparisons" for (validationType, referencename), validations in six.iteritems(comparisonLists): if issubclass(validationType, ValidationWithComparison): repMap["CompareAlignments"] += validationType.doComparison(validations) #if user wants to merge parallely and if there are valid parallel scripts, create handle for plotting job and set merge script name accordingly if options.mergeOfflineParallel and parallelMergeObjects!={}: parallelMergeObjects["continue"]=ParallelMergeJob("TkAlMergeFinal",os.path.join(path, "TkAlMergeFinal.sh"),[]) filePath = os.path.join(path, "TkAlMergeFinal.sh") #if not merging parallel, add code to create results directory and set merge script name accordingly else: repMap["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap) filePath = os.path.join(path, "TkAlMerge.sh") #filePath = os.path.join(path, "TkAlMerge.sh") theFile = open( filePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) theFile.close() os.chmod(filePath,0o755) if options.mergeOfflineParallel: return {'TkAlMerge.sh':filePath, 'parallelMergeObjects':parallelMergeObjects} else: return filePath
def createMergeScript(path, validations): if (len(validations) == 0): raise AllInOneError("Cowardly refusing to merge nothing!") repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData": "", "CompareAlignments": "", "RunExtendedOfflineValidation": "", "RunTrackSplitPlot": "", "MergeZmumuPlots": "", "CMSSW_BASE": os.environ["CMSSW_BASE"], "SCRAM_ARCH": os.environ["SCRAM_ARCH"], "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"], }) comparisonLists = { } # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s" % (validation.__class__.__name__, referenceName) validationName = validationName.split( ".%s" % GenericValidation.defaultReferenceName)[0] validationName = validationName.split("Preexisting")[-1] if validationName in comparisonLists: comparisonLists[validationName].append(validation) else: comparisonLists[validationName] = [validation] # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["haddLoop"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = ( "if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n") repMap["copyMergeScripts"] = "" repMap["mergeParallelFilePrefixes"] = "" anythingToMerge = [] for validationType in comparisonLists: for validation in comparisonLists[validationType]: if isinstance(validation, PreexistingValidation) or validation.NJobs == 1: continue if validationType not in anythingToMerge: anythingToMerge += [validationType] repMap[ "haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"]) repMap["haddLoop"] += "tmpMergeRetCode=${?}\n" repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then " "xrdcp -f " + validation.getRepMap()["finalOutputFile"] + " root://eoscms//eos/cms" + validation.getRepMap()["finalResultFile"] + "; fi\n") repMap["haddLoop"] += ( "if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then " "mergeRetCode=${tmpMergeRetCode}; fi\n") for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " $eos rm " + longName + "\n" repMap["rmUnmerged"] += ( "else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") if "OfflineValidation" in anythingToMerge: repMap["mergeOfflineParJobsScriptPath"] = os.path.join( path, "TkAlOfflineJobsMerge.C") createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"], repMap["mergeOfflineParJobsScriptPath"]) repMap["copyMergeScripts"] += ( "cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C .\n" "rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"]) if anythingToMerge: # DownloadData is the section which merges output files from parallel jobs # it uses the file TkAlOfflineJobsMerge.C repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap) else: repMap["DownloadData"] = "" if "OfflineValidation" in comparisonLists: repMap["extendedValScriptPath"] = os.path.join( path, "TkAlExtendedOfflineValidation.C") createExtendedValidationScript(comparisonLists["OfflineValidation"], repMap["extendedValScriptPath"], "OfflineValidation") repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) if "TrackSplittingValidation" in comparisonLists: repMap["trackSplitPlotScriptPath"] = \ os.path.join(path, "TkAlTrackSplitPlot.C") createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"], repMap["trackSplitPlotScriptPath"]) repMap["RunTrackSplitPlot"] = \ replaceByMap(configTemplates.trackSplitPlotExecution, repMap) if "ZMuMuValidation" in comparisonLists: repMap["mergeZmumuPlotsScriptPath"] = \ os.path.join(path, "TkAlMergeZmumuPlots.C") createMergeZmumuPlotsScript(comparisonLists["ZMuMuValidation"], repMap["mergeZmumuPlotsScriptPath"]) repMap["MergeZmumuPlots"] = \ replaceByMap(configTemplates.mergeZmumuPlotsExecution, repMap) repMap["CompareAlignments"] = "#run comparisons" if "OfflineValidation" in comparisonLists: compareStrings = [ val.getCompareStrings("OfflineValidation") for val in comparisonLists["OfflineValidation"] ] compareStringsPlain = [ val.getCompareStrings("OfflineValidation", plain=True) for val in comparisonLists["OfflineValidation"] ] repMap.update({ "validationId": "OfflineValidation", "compareStrings": " , ".join(compareStrings), "compareStringsPlain": " ".join(compareStringsPlain) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open(filePath, "w") theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap)) theFile.close() os.chmod(filePath, 0o755) return filePath
def createMergeScript( path, validations ): if(len(validations) == 0): raise AllInOneError("Cowardly refusing to merge nothing!") repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData":"", "CompareAlignments":"", "RunExtendedOfflineValidation":"", "RunTrackSplitPlot":"", "CMSSW_BASE": os.environ["CMSSW_BASE"], "SCRAM_ARCH": os.environ["SCRAM_ARCH"], "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"], }) comparisonLists = {} # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s"%(validation.__class__.__name__, referenceName) validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0] validationName = validationName.split("Preexisting")[-1] if validationName in comparisonLists: comparisonLists[ validationName ].append( validation ) else: comparisonLists[ validationName ] = [ validation ] # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["haddLoop"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n") repMap["copyMergeScripts"] = "" repMap["mergeParallelFilePrefixes"] = "" anythingToMerge = [] for validationType in comparisonLists: for validation in comparisonLists[validationType]: if isinstance(validation, PreexistingValidation) or validation.NJobs == 1: continue if validationType not in anythingToMerge: anythingToMerge += [validationType] repMap["haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"]) repMap["haddLoop"] += "tmpMergeRetCode=${?}\n" repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then " "xrdcp -f " +validation.getRepMap()["finalOutputFile"] +" root://eoscms//eos/cms" +validation.getRepMap()["finalResultFile"] +"; fi\n") repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then " "mergeRetCode=${tmpMergeRetCode}; fi\n") for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " $eos rm "+longName+"\n" repMap["rmUnmerged"] += ("else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") if "OfflineValidation" in anythingToMerge: repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C") createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"], repMap["mergeOfflineParJobsScriptPath"] ) repMap["copyMergeScripts"] += ("cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C .\n" "rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"]) if anythingToMerge: # DownloadData is the section which merges output files from parallel jobs # it uses the file TkAlOfflineJobsMerge.C repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap ) else: repMap["DownloadData"] = "" if "OfflineValidation" in comparisonLists: repMap["extendedValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C") createExtendedValidationScript(comparisonLists["OfflineValidation"], repMap["extendedValScriptPath"], "OfflineValidation") repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) if "TrackSplittingValidation" in comparisonLists: repMap["trackSplitPlotScriptPath"] = \ os.path.join(path, "TkAlTrackSplitPlot.C") createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"], repMap["trackSplitPlotScriptPath"] ) repMap["RunTrackSplitPlot"] = \ replaceByMap(configTemplates.trackSplitPlotExecution, repMap) repMap["CompareAlignments"] = "#run comparisons" for validationId in comparisonLists: compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ] repMap.update({"validationId": validationId, "compareStrings": " , ".join(compareStrings), "compareStringsPlain": " ".join(compareStringsPlain) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open( filePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) theFile.close() os.chmod(filePath,0755) return filePath
def createMergeScript(path, validations): if (len(validations) == 0): raise AllInOneError("Cowardly refusing to merge nothing!") config = validations[0].config repMap = config.getGeneral() repMap.update({ "DownloadData": "", "CompareAlignments": "", "RunValidationPlots": "", "CMSSW_BASE": os.environ["CMSSW_BASE"], "SCRAM_ARCH": os.environ["SCRAM_ARCH"], "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"], }) comparisonLists = { } # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationtype = type(validation) if isinstance(validationtype, PreexistingValidation): #find the actual validationtype for parentclass in validationtype.mro(): if not issubclass(parentclass, PreexistingValidation): validationtype = parentclass break key = (validationtype, referenceName) if key in comparisonLists: comparisonLists[key].append(validation) else: comparisonLists[key] = [validation] # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["doMerge"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = ( "if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n") repMap["beforeMerge"] = "" repMap["mergeParallelFilePrefixes"] = "" anythingToMerge = [] for (validationType, referencename), validations in comparisonLists.iteritems(): for validation in validations: if (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)): continue if validationType not in anythingToMerge: anythingToMerge += [validationType] repMap[ "doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType repMap["beforeMerge"] += validationType.doInitMerge() repMap["doMerge"] += validation.doMerge() for f in validation.getRepMap()["outputFiles"]: longName = os.path.join("/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " eos rm " + longName + "\n" repMap["rmUnmerged"] += ( "else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") if anythingToMerge: repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap) else: repMap["DownloadData"] = "" repMap["RunValidationPlots"] = "" for (validationType, referencename), validations in comparisonLists.iteritems(): if issubclass(validationType, ValidationWithPlots): repMap["RunValidationPlots"] += validationType.doRunPlots( validations) repMap["CompareAlignments"] = "#run comparisons" for (validationType, referencename), validations in comparisonLists.iteritems(): if issubclass(validationType, ValidationWithComparison): repMap["CompareAlignments"] += validationType.doComparison( validations) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open(filePath, "w") theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap)) theFile.close() os.chmod(filePath, 0o755) return filePath
def createMergeScript(path, validations, options): if (len(validations) == 0): raise AllInOneError("Cowardly refusing to merge nothing!") repMap = {} comparisonLists = { } # directory of lists containing the validations that are comparable for validation in validations: if validation.config.has_section("IOV"): iov = validation.config.get("IOV", "iov") validation.defaultReferenceName = iov for referenceName in validation.filesToCompare: validationtype = type(validation) validationName = validation.name if validation.config.has_section("IOV") and ( referenceName == "Tracker_defaultRange" or referenceName == "Tracker_autoRange"): referenceName = iov if issubclass(validationtype, PreexistingValidation): validationName = validation.originalValName #find the actual validationtype for parentclass in validationtype.mro(): if not issubclass(parentclass, PreexistingValidation): validationtype = parentclass break key = (validationtype, validationName, referenceName) if key in comparisonLists: comparisonLists[key].append(validation) else: comparisonLists[key] = [validation] repMap[key] = validation.config.getGeneral() repMap[key].update({ "DownloadData": "", "CompareAlignments": "", "RunValidationPlots": "", "CMSSW_BASE": os.environ["CMSSW_BASE"], "SCRAM_ARCH": os.environ["SCRAM_ARCH"], "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"], }) # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap[key]["doMerge"] = "mergeRetCode=0\n" repMap[key]["rmUnmerged"] = ( "if [[ mergeRetCode -eq 0 ]]; then\n" " echo -e \\n\"Merging succeeded, removing original files.\"\n" ) repMap[key]["beforeMerge"] = "" repMap[key]["mergeParallelFilePrefixes"] = "" repMap[key]["createResultsDirectory"] = "" #print("comparisonLists") #pprint.pprint(comparisonLists) anythingToMerge = [] for (validationtype, validationName, referenceName), validations in six.iteritems(comparisonLists): #pprint.pprint("validations") #pprint.pprint(validations) globalDictionaries.plottingOptions = {} lmap(lambda validation: validation.getRepMap(), validations) #plotInfo = "plots:offline" #allPlotInfo = dict(validations[0].config.items(plotInfo)) #repMap[(validationtype, validationName, referenceName)].update(allPlotInfo) for validation in validations: validation.getRepMap() #pprint.pprint("validation in validations") #pprint.pprint(validation) #parallel merging if not (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)): if (validationtype, validationName, referenceName) not in anythingToMerge: anythingToMerge.append( (validationtype, validationName, referenceName)) repMap[( validationtype, validationName, referenceName )]["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationtype.valType repMap[(validationtype, validationName, referenceName )]["beforeMerge"] += validationtype.doInitMerge() repMap[(validationtype, validationName, referenceName)]["doMerge"] += validation.doMerge() for f in validation.getRepMap()["outputFiles"]: longName = os.path.join( "/eos/cms/store/group/alca_trackeralign/AlignmentValidation/", validation.getRepMap()["eosdir"], f) repMap[(validationtype, validationName, referenceName )]["rmUnmerged"] += " rm " + longName + "\n" repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += ( "else\n" " echo -e \\n\"WARNING: Merging failed, unmerged" " files won't be deleted.\\n" "(Ignore this warning if merging was done earlier)\"\n" "fi\n") if anythingToMerge: repMap[(validationtype, validationName, referenceName)]["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap[(validationtype, validationName, referenceName)]) else: repMap[(validationtype, validationName, referenceName)]["DownloadData"] = "" repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = "" repMap[(validationtype, validationName, referenceName)]["plottingscriptpath"] = "" if issubclass(validationtype, ValidationWithPlots): repMap[( validationtype, validationName, referenceName )]["RunValidationPlots"] = validationtype.doRunPlots(validations) repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] = "#run comparisons" if issubclass(validationtype, ValidationWithComparison): repMap[( validationtype, validationName, referenceName )]["CompareAlignments"] += validationtype.doComparison(validations) #if not merging parallel, add code to create results directory and set merge script name accordingly if validations[0].config.has_section("IOV"): repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"] = replaceByMap( configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)]) filePath = os.path.join( repMap[(validationtype, validationName, referenceName)]["scriptsdir"], "TkAlMerge.sh") else: repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"] = replaceByMap( configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)]) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open(filePath, "w") theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap[(validationtype, validationName, referenceName)])) theFile.close() os.chmod(filePath, 0o755)
def createParallelMergeScript( path, validations ): if( len(validations) == 0 ): raise AllInOneError, "cowardly refusing to merge nothing!" repMap = validations[0].getRepMap() #FIXME - not nice this way repMap.update({ "DownloadData":"", "CompareAlignments":"", "RunExtendedOfflineValidation":"" }) comparisonLists = {} # directory of lists containing the validations that are comparable for validation in validations: for referenceName in validation.filesToCompare: validationName = "%s.%s"%(validation.__class__.__name__, referenceName) validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0] if validationName in comparisonLists: comparisonLists[ validationName ].append( validation ) else: comparisonLists[ validationName ] = [ validation ] if "OfflineValidationParallel" in comparisonLists: repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C") createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] ) repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C") createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"], repMap["mergeOfflineParJobsScriptPath"] ) # introduced to merge individual validation outputs separately # -> avoids problems with merge script repMap["haddLoop"] = "mergeRetCode=0\n" repMap["rmUnmerged"] = "if [[ mergeRetCode -eq 0 ]]; then\n" for validation in comparisonLists["OfflineValidationParallel"]: repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"]) repMap["haddLoop"] += "tmpMergeRetCode=${?}\n" repMap["haddLoop"] += ("if [[ mergeRetCode -eq 0 ]]; " "then mergeRetCode=${tmpMergeRetCode}; " "fi\n") repMap["haddLoop"] += ("cmsStage -f " +validation.getRepMap()["outputFile"] +" " +validation.getRepMap()["resultFile"] +"\n") for f in validation.outputFiles: longName = os.path.join("/store/caf/user/$USER/", validation.getRepMap()["eosdir"], f) repMap["rmUnmerged"] += " cmsRm "+longName+"\n" repMap["rmUnmerged"] += ("else\n" " echo \"WARNING: Merging failed, unmerged" " files won't be deleted.\"\n" "fi\n") repMap["RunExtendedOfflineValidation"] = \ replaceByMap(configTemplates.extendedValidationExecution, repMap) # DownloadData is the section which merges output files from parallel jobs # it uses the file TkAlOfflineJobsMerge.C repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap) repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap ) repMap["CompareAlignments"] = "#run comparisons" for validationId in comparisonLists: compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] repMap.update({"validationId": validationId, "compareStrings": " , ".join(compareStrings) }) repMap["CompareAlignments"] += \ replaceByMap(configTemplates.compareAlignmentsExecution, repMap) filePath = os.path.join(path, "TkAlMerge.sh") theFile = open( filePath, "w" ) theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) theFile.close() os.chmod(filePath,0755) return filePath