def createMergeScript(path, validations):
if (len(validations) == 0):
msg = "Cowardly refusing to merge nothing!"
raise AllInOneError(msg)
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData": "",
"CompareAlignments": "",
"RunExtendedOfflineValidation": ""
})
comparisonLists = {
} # directory of lists containing the validations that are comparable
resultPlotFile = "" # string of a file name for createExtendedValidationScript
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s" % (validation.__class__.__name__,
referenceName)
validationName = validationName.split(
".%s" % GenericValidation.defaultReferenceName)[0]
if validationName in comparisonLists:
comparisonLists[validationName].append(validation)
else:
comparisonLists[validationName] = [validation]
if validationName == "OfflineValidation":
resultPlotFile = validationName
if "OfflineValidation" in comparisonLists:
repMap["extendeValScriptPath"] = \
os.path.join(path, "TkAlExtendedOfflineValidation.C")
createExtendedValidationScript(comparisonLists["OfflineValidation"],
repMap["extendeValScriptPath"],
resultPlotFile)
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
repMap["CompareAlignments"] = "#run comparisons"
for validationId in comparisonLists:
compareStrings = [
val.getCompareStrings(validationId)
for val in comparisonLists[validationId]
]
repMap.update({
"validationId": validationId,
"compareStrings": " , ".join(compareStrings)
})
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open(filePath, "w")
theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap))
theFile.close()
os.chmod(filePath, 0755)
return filePath
def createMergeScript( path, validations ):
if(len(validations) == 0):
msg = "Cowardly refusing to merge nothing!"
raise AllInOneError(msg)
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData":"",
"CompareAlignments":"",
"RunExtendedOfflineValidation":"",
"RunTrackSplitPlot":""
})
comparisonLists = {} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
if validationName in comparisonLists:
comparisonLists[ validationName ].append( validation )
else:
comparisonLists[ validationName ] = [ validation ]
if "OfflineValidation" in comparisonLists:
repMap["extendeValScriptPath"] = \
os.path.join(path, "TkAlExtendedOfflineValidation.C")
createExtendedValidationScript(comparisonLists["OfflineValidation"],
repMap["extendeValScriptPath"],
"OfflineValidation")
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
if "TrackSplittingValidation" in comparisonLists:
repMap["trackSplitPlotScriptPath"] = \
os.path.join(path, "TkAlTrackSplitPlot.C")
createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
repMap["trackSplitPlotScriptPath"] )
repMap["RunTrackSplitPlot"] = \
replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
repMap["CompareAlignments"] = "#run comparisons"
for validationId in comparisonLists:
compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ]
repMap.update({"validationId": validationId,
"compareStrings": " , ".join(compareStrings),
"compareStringsPlain": " ".join(compareStringsPlain) })
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open( filePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
theFile.close()
os.chmod(filePath,0755)
return filePath
def createParallelMergeScript( path, validations ):
if( len(validations) == 0 ):
raise AllInOneError, "cowardly refusing to merge nothing!"
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData":"",
"CompareAlignments":"",
"RunExtendedOfflineValidation":""
})
comparisonLists = {} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
if validationName in comparisonLists:
comparisonLists[ validationName ].append( validation )
else:
comparisonLists[ validationName ] = [ validation ]
if "OfflineValidationParallel" in comparisonLists:
repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] )
repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"], repMap["mergeOfflineParJobsScriptPath"] )
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
# needed to copy the right merged file back to eos
repMap["resultFile"] = comparisonLists["OfflineValidationParallel"][0].getRepMap()["resultFile"]
repMap["outputFile"] = comparisonLists["OfflineValidationParallel"][0].getRepMap()["outputFile"]
# DownloadData is the section which merges output files from parallel jobs
# it uses the file TkAlOfflineJobsMerge.C
repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
repMap["CompareAlignments"] = "#run comparisons"
for validationId in comparisonLists:
compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
repMap.update({"validationId": validationId,
"compareStrings": " , ".join(compareStrings) })
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open( filePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
theFile.close()
os.chmod(filePath,0755)
return filePath
def submitjob(jobtext, jobname=None, jobtime=None, queue=None, interactive=False, waitids=[], waitsuccessids=[], outputfile=None, errorfile=None, docd=False, morerepmap=None, email=True):
if not email: raise RuntimeError("Not sure how to turn off email for lxplus")
if outputfile is not None:
outputfile = outputfile.format(jobid="%J")
if errorfile is not None:
errorfile = errorfile.format(jobid="%J")
if interactive: queue = "cmsinter"
if queue is None:
if jobtime is None:
raise ValueError("Have to provide either queue or jobtime (or set interactive=True)!")
#http://stackoverflow.com/a/4628148/5228524
match = re.match(r'((?P<days>\d+?)-)?((?P<hours>\d+?):)?((?P<minutes>\d+?):)?((?P<seconds>\d+?))?', jobtime)
if not match: raise ValueError("Bad time string {}".format(jobtime))
kwargs = {key: int(value) for key, value in match.groupdict().iteritems() if value is not None}
delta = timedelta(**kwargs)
if not delta: raise ValueError("Job takes no time! {}".format(jobtime))
if delta <= timedelta(minutes=8): queue = "8nm"
elif delta <= timedelta(hours=1): queue = "1nh"
elif delta <= timedelta(hours=8): queue = "8nh"
elif delta <= timedelta(days=1): queue = "1nd"
elif delta <= timedelta(days=2): queue = "2nd"
elif delta <= timedelta(weeks=1): queue = "1nw"
elif delta <= timedelta(weeks=2): queue = "2nw"
else: raise ValueError("time {} is more than 2 weeks!".format(jobtime))
run = """
echo .oO[jobtext]Oo. | bsub .oO[options]Oo.
"""
optionsdict = {
"-q": queue,
"-J": jobname,
"-o": outputfile,
"-e": errorfile,
"-w": " && ".join(["ended({:d})".format(id) for id in waitids]+["done({:d})".format(id) for id in waitsuccessids])
}
repmap = {
"jobtext": quote("cd .oO[CMSSW_BASE]Oo. && eval $(scram ru -sh) && .oO[docd]Oo. && "+jobtext),
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"pwd": os.getcwd(),
"options": " ".join("{} {}".format(key, quote(value)) for key, value in optionsdict.iteritems() if value),
"docd": "cd .oO[pwd]Oo." if docd else "cd -"
}
if morerepmap:
assert not (set(repmap) & set(morerepmap))
repmap.update(morerepmap)
if interactive:
repmap["options"] += " -I"
subprocess.check_call(replaceByMap(run, repmap), shell=True)
else:
bsubout = subprocess.check_output(replaceByMap(run, repmap), shell=True)
jobid = int(bsubout.split("<")[1].split(">")[0])
print bsubout,
return jobid
def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
theFile.close()
def createOfflineParJobsMergeScript(offlineValidationList, outFilePath):
repMap = offlineValidationList[0].getRepMap(
) # bit ugly since some special features are filled
theFile = open(outFilePath, "w")
theFile.write(
replaceByMap(configTemplates.mergeOfflineParJobsTemplate, repMap))
theFile.close()
def createExtendedValidationScript(offlineValidationList, outFilePath):
repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in offlineValidationList:
repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
theFile.close()
def createMergeZmumuPlotsScript(zMuMuValidationList, outFilePath):
repMap = zMuMuValidationList[0].getRepMap() # bit ugly since some special features are filled
repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
repMap[ "mergeZmumuPlotsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in zMuMuValidationList:
repMap[ "mergeZmumuPlotsInstantiation" ] = validation.appendToExtendedValidation( repMap[ "mergeZmumuPlotsInstantiation" ] )
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeZmumuPlotsTemplate ,repMap ) )
theFile.close()
def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in offlineValidationList:
repMap[ "mergeOfflinParJobsInstantiation" ] = validation.appendToMergeParJobs( repMap[ "mergeOfflinParJobsInstantiation" ] )
# validationsSoFar = 'PlotAlignmentValidation p("%(resultFile)s", "%(name)s", %(color)s, %(style)s);\n'%repMap
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
theFile.close()
def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
repMap = trackSplittingValidationList[0].getRepMap() # bit ugly since some special features are filled
repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in trackSplittingValidationList:
repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
theFile = open( outFilePath, "w" )
# theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
theFile.close()
def createMergeZmumuPlotsScript(zMuMuValidationList, outFilePath):
config = zMuMuValidationList[0].config
repMap = PlottingOptions(config, "zmumu")
repMap[ "mergeZmumuPlotsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in zMuMuValidationList:
repMap[ "mergeZmumuPlotsInstantiation" ] = validation.appendToExtendedValidation( repMap[ "mergeZmumuPlotsInstantiation" ] )
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeZmumuPlotsTemplate ,repMap ) )
theFile.close()
def createPrimaryVertexPlotScript(PrimaryVertexValidationList, outFilePath):
repMap = PrimaryVertexValidationList[0].getRepMap() # bit ugly since some special features are filled
repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
repMap[ "PrimaryVertexPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in PrimaryVertexValidationList:
repMap[ "PrimaryVertexPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "PrimaryVertexPlotInstantiation" ] )
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.PrimaryVertexPlotTemplate ,repMap ) )
theFile.close()
def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
config = trackSplittingValidationList[0].config
repMap = PlottingOptions(config, "split")
repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in trackSplittingValidationList:
repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
theFile.close()
def createExtendedValidationScript(offlineValidationList, outFilePath):
repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in offlineValidationList:
repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
theFile = open( outFilePath, "w" )
# theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
theFile.close()
def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
config = offlineValidationList[0].config
repMap = PlottingOptions(config, "offline")
repMap[ "resultPlotFile" ] = resultPlotFile
repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
for validation in offlineValidationList:
repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
theFile.close()
def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
repMap = offlineValidationList[0].getRepMap(
) # bit ugly since some special features are filled
repMap[
"mergeOfflinParJobsInstantiation"] = "" #give it a "" at first in order to get the initialisation back
for validation in offlineValidationList:
repMap[
"mergeOfflinParJobsInstantiation"] = validation.appendToMergeParJobs(
repMap["mergeOfflinParJobsInstantiation"])
# validationsSoFar = 'PlotAlignmentValidation p("%(resultFile)s", "%(name)s", %(color)s, %(style)s);\n'%repMap
theFile = open(outFilePath, "w")
theFile.write(
replaceByMap(configTemplates.mergeOfflineParJobsTemplate, repMap))
theFile.close()
def createOfflineParJobsMergeScript(offlineValidationList, outFilePath):
repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
theFile = open( outFilePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
theFile.close()
def createMergeScript(path, validations, options):
if (len(validations) == 0):
raise AllInOneError("Cowardly refusing to merge nothing!")
config = validations[0].config
repMap = config.getGeneral()
repMap.update({
"DownloadData": "",
"CompareAlignments": "",
"RunValidationPlots": "",
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"SCRAM_ARCH": os.environ["SCRAM_ARCH"],
"CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
})
comparisonLists = {
} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationtype = type(validation)
if issubclass(validationtype, PreexistingValidation):
#find the actual validationtype
for parentclass in validationtype.mro():
if not issubclass(parentclass, PreexistingValidation):
validationtype = parentclass
break
key = (validationtype, referenceName)
if key in comparisonLists:
comparisonLists[key].append(validation)
else:
comparisonLists[key] = [validation]
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["doMerge"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = (
"if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n")
repMap["beforeMerge"] = ""
repMap["mergeParallelFilePrefixes"] = ""
repMap["createResultsDirectory"] = ""
anythingToMerge = []
#prepare dictionary containing handle objects for parallel merge batch jobs
if options.mergeOfflineParallel:
parallelMergeObjects = {}
for (validationType,
referencename), validations in six.iteritems(comparisonLists):
for validation in validations:
#parallel merging
if (isinstance(validation, PreexistingValidation)
or validation.NJobs == 1
or not isinstance(validation, ParallelValidation)):
continue
if options.mergeOfflineParallel and validationType.valType == 'offline' and validation.jobmode.split(
",")[0] == "lxBatch":
repMapTemp = repMap.copy()
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
#create init script
fileName = "TkAlMergeInit"
filePath = os.path.join(path, fileName + ".sh")
theFile = open(filePath, "w")
repMapTemp["createResultsDirectory"] = "#!/bin/bash"
repMapTemp["createResultsDirectory"] += replaceByMap(
configTemplates.createResultsDirectoryTemplate,
repMapTemp)
theFile.write(
replaceByMap(
configTemplates.createResultsDirectoryTemplate,
repMapTemp))
theFile.close()
os.chmod(filePath, 0o755)
#create handle
parallelMergeObjects["init"] = ParallelMergeJob(
fileName, filePath, [])
#clear 'create result directory' code
repMapTemp["createResultsDirectory"] = ""
#edit repMapTmp as necessary:
#fill contents of mergeParallelResults
repMapTemp["beforeMerge"] += validationType.doInitMerge()
repMapTemp[
"doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs with alignment %s"\n\n' % (
validationType.valType,
validation.alignmentToValidate.name)
repMapTemp["doMerge"] += validation.doMerge()
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/eos/cms/store/caf/user/$USER/",
validation.getRepMap()["eosdir"],
f)
repMapTemp["rmUnmerged"] += " rm " + longName + "\n"
repMapTemp["rmUnmerged"] += (
"else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
#fill mergeParallelResults area of mergeTemplate
repMapTemp["DownloadData"] = replaceByMap(
configTemplates.mergeParallelResults, repMapTemp)
#fill runValidationPlots area of mergeTemplate
repMapTemp["RunValidationPlots"] = validationType.doRunPlots(
validations)
#create script file
fileName = "TkAlMergeOfflineValidation" + validation.name + validation.alignmentToValidate.name
filePath = os.path.join(path, fileName + ".sh")
theFile = open(filePath, "w")
theFile.write(
replaceByMap(configTemplates.mergeParallelOfflineTemplate,
repMapTemp))
theFile.close()
os.chmod(filePath, 0o755)
#create handle object
if "parallel" in parallelMergeObjects:
parallelMergeObjects["parallel"].append(
ParallelMergeJob(fileName, filePath, []))
else:
parallelMergeObjects["parallel"] = [
ParallelMergeJob(fileName, filePath, [])
]
continue
else:
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
repMap[
"doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType
repMap["beforeMerge"] += validationType.doInitMerge()
repMap["doMerge"] += validation.doMerge()
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/eos/cms/store/caf/user/$USER/",
validation.getRepMap()["eosdir"],
f)
repMap["rmUnmerged"] += " rm " + longName + "\n"
repMap["rmUnmerged"] += (
"else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
if anythingToMerge:
repMap["DownloadData"] += replaceByMap(
configTemplates.mergeParallelResults, repMap)
else:
repMap["DownloadData"] = ""
repMap["RunValidationPlots"] = ""
for (validationType,
referencename), validations in six.iteritems(comparisonLists):
if issubclass(validationType, ValidationWithPlots):
repMap["RunValidationPlots"] += validationType.doRunPlots(
validations)
repMap["CompareAlignments"] = "#run comparisons"
for (validationType,
referencename), validations in six.iteritems(comparisonLists):
if issubclass(validationType, ValidationWithComparison):
repMap["CompareAlignments"] += validationType.doComparison(
validations)
#if user wants to merge parallely and if there are valid parallel scripts, create handle for plotting job and set merge script name accordingly
if options.mergeOfflineParallel and parallelMergeObjects != {}:
parallelMergeObjects["continue"] = ParallelMergeJob(
"TkAlMergeFinal", os.path.join(path, "TkAlMergeFinal.sh"), [])
filePath = os.path.join(path, "TkAlMergeFinal.sh")
#if not merging parallel, add code to create results directory and set merge script name accordingly
else:
repMap["createResultsDirectory"] = replaceByMap(
configTemplates.createResultsDirectoryTemplate, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
#filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open(filePath, "w")
theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap))
theFile.close()
os.chmod(filePath, 0o755)
if options.mergeOfflineParallel:
return {
'TkAlMerge.sh': filePath,
'parallelMergeObjects': parallelMergeObjects
}
else:
return filePath
def submitjob(jobtext, jobname=None, jobtime=None, queue=None, interactive=False, waitids=[], outputfile=None, errorfile=None, docd=False, morerepmap=None, email=False, memory="3000M"):
if queue is None:
queue = "shared"
if outputfile is not None:
outputfile = outputfile.format(jobid="%j")
if errorfile is not None:
errorfile = errorfile.format(jobid="%j")
jobtemplate = textwrap.dedent("""
#!/bin/bash
. /work-zfs/lhc/cms/cmsset_default.sh &&
cd .oO[CMSSW_BASE]Oo. &&
eval $(scram ru -sh) &&
.oO[docd]Oo. &&
echo "SLURM job running in: " `pwd` &&
#commands
.oO[jobtext]Oo.
""").strip()
repmap = {
"jobtext": jobtext,
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"pwd": os.getcwd(),
"jobname": jobname,
"jobtime": jobtime,
"queue": queue,
"waitids": ":".join("{:d}".format(id) for id in waitids),
"docd": "cd .oO[pwd]Oo." if docd else "cd $(mktemp -d)",
}
if morerepmap:
assert not (set(repmap) & set(morerepmap))
repmap.update(morerepmap)
options = {
"--job-name": ".oO[jobname]Oo.",
"--time": ".oO[jobtime]Oo.",
"--nodes": "1",
"--ntasks-per-node": "1",
"--partition": ".oO[queue]Oo.",
"--mem": memory,
"--output": outputfile,
"--error": errorfile,
}
if waitids:
options["--dependency"] = "afterany:.oO[waitids]Oo."
if email:
options["--mail-user"] = email
options["--mail-type"] = "end"
options = {replaceByMap(k, repmap): replaceByMap(v, repmap) for k, v in options.iteritems() if v is not None}
options = ["{}={}".format(k, quote(v)) for k, v in options.iteritems()]
if interactive:
subprocess.check_call(
["srun"] + options +
["bash", "-c", replaceByMap(".oO[jobtext]Oo.", repmap)]
)
else:
with tempfile.NamedTemporaryFile(bufsize=0) as f:
f.write(replaceByMap(jobtemplate, repmap))
os.chmod(f.name, os.stat(f.name).st_mode | stat.S_IEXEC)
sbatchout = subprocess.check_output(["sbatch"] + options + [f.name])
jobid = int(sbatchout.split()[-1])
print sbatchout,
return jobid
def runcombine(analysis, foldername, **kwargs):
usechannels = channels
expectvalues = [0.0]
plotname = "limit"
legendposition = (.2, .7, .6, .9)
CLtextposition = "left"
productions = config.productionsforcombine
usesystematics = True
usebkg = True
runobs = True
defaultscanrange = scanrange = (-1.0, 1.0)
defaultnpoints = npoints = 101
defaultscenario = scenario = 1
scalemuvmuftogether = False
for kw, kwarg in kwargs.iteritems():
if kw == "channels":
usechannels = [Channel(c) for c in kwarg.split(",")]
elif kw == "expectvalues":
try:
expectvalues = [fai if fai=="minimum" else float(fai) for fai in kwarg.split(",")]
except ValueError:
if kwarg == "":
expectvalues = []
else:
raise ValueError("expectvalues has to contain floats separated by commas!")
elif kw == "plotname":
plotname = kwarg
elif kw == "CLtextposition":
CLtextposition = kwarg
elif kw == "legendposition":
try:
legendposition = [float(a) for a in kwarg.split(",")]
if len(legendposition) != 4: raise ValueError
except ValueError:
raise ValueError("legendposition has to contain 4 floats separated by commas!")
elif kw == "productions":
productions = [Production(p) for p in kwarg.split(",")]
elif kw == "usesystematics":
usesystematics = bool(int(kwarg))
elif kw == "usebkg":
usebkg = bool(int(kwarg))
elif kw == "scanrange":
try:
scanrange = tuple(float(a) for a in kwarg.split(","))
if len(scanrange) != 2: raise ValueError
except ValueError:
raise ValueError("scanrange has to contain 2 floats separated by a comma!")
elif kw == "npoints":
npoints = int(kwarg)
elif kw == "scenario":
scenario = int(kwarg)
elif kw == "scalemuvmuftogether":
scalemuvmuftogether = bool(int(kwarg))
elif kw == "runobs":
runobs = bool(int(kwarg))
else:
raise TypeError("Unknown kwarg: {}".format(kw))
if not config.unblindscans:
luminosity = float(Luminosity("forexpectedscan"))
else:
luminosity = None
years = [p.year for p in productions]
if len(set(years)) != len(years):
raise ValueError("Some of your productions are from the same year!")
workspacefileappend = ""
if scalemuvmuftogether:
workspacefileappend += "_scalemuvmuftogether"
print workspacefileappend
moreappend = ".oO[workspacefileappend]Oo."
if not usesystematics:
moreappend += "_nosystematics"
if not (npoints == defaultnpoints and scanrange == defaultscanrange):
moreappend += "_{},{},{}".format(npoints, *scanrange)
analysis = Analysis(analysis)
foldername = "{}_{}".format(analysis, foldername)
if scenario != defaultscenario:
foldername += "_scenario{}".format(scenario)
repmap = {
"foldername": pipes.quote(foldername),
"analysis": str(analysis),
"cardstocombine": " ".join("hzz4l_{}S_{}.txt".format(channel, production.year) for channel, production in product(usechannels, productions)),
"workspacefile": "floatMu.oO[workspacefileappend]Oo..root",
"filename": "higgsCombine_.oO[append]Oo..oO[moreappend]Oo..MultiDimFit.mH125.root",
"expectedappend": "exp_.oO[expectfai]Oo.",
"observedappend": "obs",
"workspacefileappend": workspacefileappend,
"usesystematics": str(int(usesystematics)),
"moreappend": moreappend,
"npoints": str(npoints),
"scanrange": "{},{}".format(*scanrange),
"SM_XS_VV": str(analysis.SM_XS_VV),
"BSM_XS_VV": str(analysis.BSM_XS_VV),
"int_XS_VV": str(analysis.int_XS_VV),
"morePO": "--PO scalemuvmuftogether" if scalemuvmuftogether else "",
}
with filemanager.cd(os.path.join(config.repositorydir, "CMSSW_8_1_0/src/HiggsAnalysis/HZZ4l_Combination/CreateDatacards")):
for production in productions:
production = Production(production)
if not all(os.path.exists("cards_{}/HCG/125/{}/hzz4l_{}S_{}.input.root".format(foldername, production.year, channel, production.year)) for channel in usechannels):
makeworkspacesmap = repmap.copy()
makeworkspacesmap["production"] = str(production)
subprocess.check_call(replaceByMap(makeworkspacestemplate, makeworkspacesmap), shell=True)
with open("cards_{}/.gitignore".format(foldername), "w") as f:
f.write("*")
with filemanager.cd("cards_{}/HCG/125/{}".format(foldername, production.year)):
#replace rates
for channel, production in product(channels, productions):
if channel in usechannels:
with open("hzz4l_{}S_{}.txt".format(channel, production.year)) as f:
contents = f.read()
if "\n#rate" in contents: continue #already did this
for line in contents.split("\n"):
if line.startswith("rate"):
if config.unblindscans:
rates = getrates(channel, "fordata", production, usebkg=usebkg)
else:
rates = getrates(channel, "forexpectedscan", production, usebkg=usebkg)
contents = contents.replace(line, "#"+line+"\n"+rates)
break
with open("hzz4l_{}S_{}.txt".format(channel, production.year), "w") as f:
f.write(contents)
else:
if os.path.exists("hzz4l_{}S_{}.txt".format(channel, production.year)):
os.remove("hzz4l_{}S_{}.txt".format(channel, production.year))
os.remove("hzz4l_{}S_{}.input.root".format(channel, production.year))
if not os.path.exists(replaceByMap(".oO[workspacefile]Oo.", repmap)):
for channel, production in product(usechannels, productions):
replacesystematics(channel, production, scenario=scenario, luminosity=luminosity)
subprocess.check_call(replaceByMap(createworkspacetemplate, repmap), shell=True)
if config.unblindscans and runobs:
repmap_obs = repmap.copy()
repmap_obs["expectfai"] = "0.0" #starting point
repmap_obs["append"] = ".oO[observedappend]Oo."
if not os.path.exists(replaceByMap(".oO[filename]Oo.", repmap_obs)):
subprocess.check_call(replaceByMap(observationcombinetemplate, repmap_obs), shell=True)
f = ROOT.TFile(replaceByMap(".oO[filename]Oo.", repmap_obs))
minimum = (float("nan"), float("inf"))
for entry in f.limit:
if f.limit.deltaNLL < minimum[1]:
minimum = (f.limit.CMS_zz4l_fai1, f.limit.deltaNLL)
minimum = minimum[0]
del f
for expectfai in expectvalues:
repmap_exp = repmap.copy()
if expectfai == "minimum":
expectfai = minimum
repmap_exp["expectfai"] = str(expectfai)
repmap_exp["append"] = ".oO[expectedappend]Oo."
if not os.path.exists(replaceByMap(".oO[filename]Oo.", repmap_exp)):
subprocess.check_call(replaceByMap(runcombinetemplate, repmap_exp), shell=True)
saveasdir = os.path.join(config.plotsbasedir, "limits", foldername)
try:
os.makedirs(saveasdir)
except OSError:
pass
plotscans = []
if config.unblindscans and runobs:
plotscans.append("obs")
for expectfai in expectvalues:
if expectfai == "minimum":
expectfai = minimum
plotscans.append(expectfai)
for ext in "png eps root pdf".split():
plotname = plotname.replace("."+ext, "")
plotname += moreappend
plotname = replaceByMap(plotname, repmap)
plotlimits(os.path.join(saveasdir, plotname), analysis, *plotscans, productions=productions, legendposition=legendposition, CLtextposition=CLtextposition, moreappend=replaceByMap(moreappend, repmap))
with open(os.path.join(saveasdir, plotname+".txt"), "w") as f:
f.write(" ".join(["python"]+sys.argv))
f.write("\n\n\n")
f.write("python limits.py ")
for arg in sys.argv[1:]:
if "=" in arg: continue
f.write(arg+" ")
f.write("plotname="+plotname+" ")
f.write("\n")
def createParallelMergeScript(path, validations):
if (len(validations) == 0):
raise AllInOneError, "cowardly refusing to merge nothing!"
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData": "",
"CompareAlignments": "",
"RunExtendedOfflineValidation": "",
"RunTrackSplitPlot": ""
})
comparisonLists = {
} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s" % (validation.__class__.__name__,
referenceName)
validationName = validationName.split(
".%s" % GenericValidation.defaultReferenceName)[0]
if validationName in comparisonLists:
comparisonLists[validationName].append(validation)
else:
comparisonLists[validationName] = [validation]
if "OfflineValidationParallel" in comparisonLists:
repMap["extendeValScriptPath"] = os.path.join(
path, "TkAlExtendedOfflineValidation.C")
repMap["mergeOfflineParJobsScriptPath"] = os.path.join(
path, "TkAlOfflineJobsMerge.C")
createOfflineJobsMergeScript(
comparisonLists["OfflineValidationParallel"],
repMap["mergeOfflineParJobsScriptPath"])
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["haddLoop"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = (
"if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n")
for validation in comparisonLists["OfflineValidationParallel"]:
repMap["haddLoop"] = validation.appendToMergeParJobs(
repMap["haddLoop"])
repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
"cmsStage -f " +
validation.getRepMap()["outputFile"] + " " +
validation.getRepMap()["resultFile"] +
"; fi\n")
repMap["haddLoop"] += (
"if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
"mergeRetCode=${tmpMergeRetCode}; fi\n")
for f in validation.outputFiles:
longName = os.path.join("/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMap["rmUnmerged"] += " cmsRm " + longName + "\n"
repMap["rmUnmerged"] += (
"else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
# DownloadData is the section which merges output files from parallel jobs
# it uses the file TkAlOfflineJobsMerge.C
repMap["DownloadData"] += replaceByMap(
"rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
repMap["DownloadData"] += replaceByMap(
configTemplates.mergeOfflineParallelResults, repMap)
# For the rest of the script, OfflineValidations and
# OfflineParallelValidations are comparable:
if "OfflineValidation" in comparisonLists:
comparisonLists["OfflineValidationParallel"].extend(
comparisonLists["OfflineValidation"])
del comparisonLists["OfflineValidation"]
createExtendedValidationScript(
comparisonLists["OfflineValidationParallel"],
repMap["extendeValScriptPath"], "OfflineValidationParallel")
repMap["CompareAlignments"] = "#run comparisons"
for validationId in comparisonLists:
compareStrings = [
val.getCompareStrings(validationId)
for val in comparisonLists[validationId]
]
compareStringsPlain = [
val.getCompareStrings(validationId, plain=True)
for val in comparisonLists[validationId]
]
repMap.update({
"validationId": validationId,
"compareStrings": " , ".join(compareStrings),
"compareStringsPlain": " ".join(compareStringsPlain)
})
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open(filePath, "w")
theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap))
theFile.close()
os.chmod(filePath, 0755)
return filePath
def createMergeScript( path, validations ):
if(len(validations) == 0):
raise AllInOneError("Cowardly refusing to merge nothing!")
config = validations[0].config
repMap = config.getGeneral()
repMap.update({
"DownloadData":"",
"CompareAlignments":"",
"RunValidationPlots":"",
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"SCRAM_ARCH": os.environ["SCRAM_ARCH"],
"CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
})
comparisonLists = {} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationtype = type(validation)
if isinstance(validationtype, PreexistingValidation):
#find the actual validationtype
for parentclass in validationtype.mro():
if not issubclass(parentclass, PreexistingValidation):
validationtype = parentclass
break
key = (validationtype, referenceName)
if key in comparisonLists:
comparisonLists[key].append(validation)
else:
comparisonLists[key] = [validation]
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["doMerge"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n")
repMap["beforeMerge"] = ""
repMap["mergeParallelFilePrefixes"] = ""
anythingToMerge = []
for (validationType, referencename), validations in comparisonLists.iteritems():
for validation in validations:
if (isinstance(validation, PreexistingValidation)
or validation.NJobs == 1
or not isinstance(validation, ParallelValidation)):
continue
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
repMap["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType
repMap["beforeMerge"] += validationType.doInitMerge()
repMap["doMerge"] += validation.doMerge()
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMap["rmUnmerged"] += " $eos rm "+longName+"\n"
repMap["rmUnmerged"] += ("else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
if anythingToMerge:
repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
else:
repMap["DownloadData"] = ""
repMap["RunValidationPlots"] = ""
for (validationType, referencename), validations in comparisonLists.iteritems():
if issubclass(validationType, ValidationWithPlots):
repMap["RunValidationPlots"] += validationType.doRunPlots(validations)
repMap["CompareAlignments"] = "#run comparisons"
for (validationType, referencename), validations in comparisonLists.iteritems():
if issubclass(validationType, ValidationWithComparison):
repMap["CompareAlignments"] += validationType.doComparison(validations)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open( filePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
theFile.close()
os.chmod(filePath,0o755)
return filePath
def createParallelMergeScript(path, validations):
if (len(validations) == 0):
raise AllInOneError, "cowardly refusing to merge nothing!"
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData": "",
"CompareAlignments": "",
"RunExtendedOfflineValidation": ""
})
comparisonLists = {
} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s" % (validation.__class__.__name__,
referenceName)
validationName = validationName.split(
".%s" % GenericValidation.defaultReferenceName)[0]
if validationName in comparisonLists:
comparisonLists[validationName].append(validation)
else:
comparisonLists[validationName] = [validation]
if "OfflineValidationParallel" in comparisonLists:
repMap["extendeValScriptPath"] = os.path.join(
path, "TkAlExtendedOfflineValidation.C")
createExtendedValidationScript(
comparisonLists["OfflineValidationParallel"],
repMap["extendeValScriptPath"])
repMap["mergeOfflineParJobsScriptPath"] = os.path.join(
path, "TkAlOfflineJobsMerge.C")
createOfflineJobsMergeScript(
comparisonLists["OfflineValidationParallel"],
repMap["mergeOfflineParJobsScriptPath"])
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
# needed to copy the right merged file back to eos
repMap["resultFile"] = comparisonLists["OfflineValidationParallel"][
0].getRepMap()["resultFile"]
repMap["outputFile"] = comparisonLists["OfflineValidationParallel"][
0].getRepMap()["outputFile"]
# DownloadData is the section which merges output files from parallel jobs
# it uses the file TkAlOfflineJobsMerge.C
repMap["DownloadData"] += replaceByMap(
"rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
repMap["DownloadData"] += replaceByMap(
configTemplates.mergeOfflineParallelResults, repMap)
repMap["CompareAlignments"] = "#run comparisons"
for validationId in comparisonLists:
compareStrings = [
val.getCompareStrings(validationId)
for val in comparisonLists[validationId]
]
repMap.update({
"validationId": validationId,
"compareStrings": " , ".join(compareStrings)
})
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open(filePath, "w")
theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap))
theFile.close()
os.chmod(filePath, 0755)
return filePath
def createMergeScript( path, validations, options ):
if(len(validations) == 0):
raise AllInOneError("Cowardly refusing to merge nothing!")
config = validations[0].config
repMap = config.getGeneral()
repMap.update({
"DownloadData":"",
"CompareAlignments":"",
"RunValidationPlots":"",
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"SCRAM_ARCH": os.environ["SCRAM_ARCH"],
"CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
})
comparisonLists = {} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationtype = type(validation)
if issubclass(validationtype, PreexistingValidation):
#find the actual validationtype
for parentclass in validationtype.mro():
if not issubclass(parentclass, PreexistingValidation):
validationtype = parentclass
break
key = (validationtype, referenceName)
if key in comparisonLists:
comparisonLists[key].append(validation)
else:
comparisonLists[key] = [validation]
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["doMerge"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n")
repMap["beforeMerge"] = ""
repMap["mergeParallelFilePrefixes"] = ""
repMap["createResultsDirectory"]=""
anythingToMerge = []
#prepare dictionary containing handle objects for parallel merge batch jobs
if options.mergeOfflineParallel:
parallelMergeObjects={}
for (validationType, referencename), validations in six.iteritems(comparisonLists):
for validation in validations:
#parallel merging
if (isinstance(validation, PreexistingValidation)
or validation.NJobs == 1
or not isinstance(validation, ParallelValidation)):
continue
if options.mergeOfflineParallel and validationType.valType=='offline' and validation.jobmode.split(",")[0]=="lxBatch":
repMapTemp=repMap.copy()
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
#create init script
fileName="TkAlMergeInit"
filePath = os.path.join(path, fileName+".sh")
theFile = open( filePath, "w" )
repMapTemp["createResultsDirectory"]="#!/bin/bash"
repMapTemp["createResultsDirectory"]+=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMapTemp)
theFile.write( replaceByMap( configTemplates.createResultsDirectoryTemplate, repMapTemp ) )
theFile.close()
os.chmod(filePath,0o755)
#create handle
parallelMergeObjects["init"]=ParallelMergeJob(fileName, filePath,[])
#clear 'create result directory' code
repMapTemp["createResultsDirectory"]=""
#edit repMapTmp as necessary:
#fill contents of mergeParallelResults
repMapTemp["beforeMerge"] += validationType.doInitMerge()
repMapTemp["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs with alignment %s"\n\n' % (validationType.valType,validation.alignmentToValidate.name)
repMapTemp["doMerge"] += validation.doMerge()
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/eos/cms/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMapTemp["rmUnmerged"] += " rm "+longName+"\n"
repMapTemp["rmUnmerged"] += ("else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
#fill mergeParallelResults area of mergeTemplate
repMapTemp["DownloadData"] = replaceByMap( configTemplates.mergeParallelResults, repMapTemp )
#fill runValidationPlots area of mergeTemplate
repMapTemp["RunValidationPlots"] = validationType.doRunPlots(validations)
#create script file
fileName="TkAlMerge"+validation.alignmentToValidate.name
filePath = os.path.join(path, fileName+".sh")
theFile = open( filePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeParallelOfflineTemplate, repMapTemp ) )
theFile.close()
os.chmod(filePath,0o755)
#create handle object
if "parallel" in parallelMergeObjects:
parallelMergeObjects["parallel"].append(ParallelMergeJob(fileName, filePath,[]))
else:
parallelMergeObjects["parallel"]=[ParallelMergeJob(fileName, filePath,[])]
continue
else:
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
repMap["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType
repMap["beforeMerge"] += validationType.doInitMerge()
repMap["doMerge"] += validation.doMerge()
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/eos/cms/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMap["rmUnmerged"] += " rm "+longName+"\n"
repMap["rmUnmerged"] += ("else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
if anythingToMerge:
repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
else:
repMap["DownloadData"] = ""
repMap["RunValidationPlots"] = ""
for (validationType, referencename), validations in six.iteritems(comparisonLists):
if issubclass(validationType, ValidationWithPlots):
repMap["RunValidationPlots"] += validationType.doRunPlots(validations)
repMap["CompareAlignments"] = "#run comparisons"
for (validationType, referencename), validations in six.iteritems(comparisonLists):
if issubclass(validationType, ValidationWithComparison):
repMap["CompareAlignments"] += validationType.doComparison(validations)
#if user wants to merge parallely and if there are valid parallel scripts, create handle for plotting job and set merge script name accordingly
if options.mergeOfflineParallel and parallelMergeObjects!={}:
parallelMergeObjects["continue"]=ParallelMergeJob("TkAlMergeFinal",os.path.join(path, "TkAlMergeFinal.sh"),[])
filePath = os.path.join(path, "TkAlMergeFinal.sh")
#if not merging parallel, add code to create results directory and set merge script name accordingly
else:
repMap["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
#filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open( filePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
theFile.close()
os.chmod(filePath,0o755)
if options.mergeOfflineParallel:
return {'TkAlMerge.sh':filePath, 'parallelMergeObjects':parallelMergeObjects}
else:
return filePath
def createMergeScript(path, validations):
if (len(validations) == 0):
raise AllInOneError("Cowardly refusing to merge nothing!")
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData": "",
"CompareAlignments": "",
"RunExtendedOfflineValidation": "",
"RunTrackSplitPlot": "",
"MergeZmumuPlots": "",
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"SCRAM_ARCH": os.environ["SCRAM_ARCH"],
"CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
})
comparisonLists = {
} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s" % (validation.__class__.__name__,
referenceName)
validationName = validationName.split(
".%s" % GenericValidation.defaultReferenceName)[0]
validationName = validationName.split("Preexisting")[-1]
if validationName in comparisonLists:
comparisonLists[validationName].append(validation)
else:
comparisonLists[validationName] = [validation]
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["haddLoop"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = (
"if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n")
repMap["copyMergeScripts"] = ""
repMap["mergeParallelFilePrefixes"] = ""
anythingToMerge = []
for validationType in comparisonLists:
for validation in comparisonLists[validationType]:
if isinstance(validation,
PreexistingValidation) or validation.NJobs == 1:
continue
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
repMap[
"haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType
repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"])
repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
"xrdcp -f " +
validation.getRepMap()["finalOutputFile"] +
" root://eoscms//eos/cms" +
validation.getRepMap()["finalResultFile"] +
"; fi\n")
repMap["haddLoop"] += (
"if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
"mergeRetCode=${tmpMergeRetCode}; fi\n")
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMap["rmUnmerged"] += " $eos rm " + longName + "\n"
repMap["rmUnmerged"] += (
"else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
if "OfflineValidation" in anythingToMerge:
repMap["mergeOfflineParJobsScriptPath"] = os.path.join(
path, "TkAlOfflineJobsMerge.C")
createOfflineParJobsMergeScript(
comparisonLists["OfflineValidation"],
repMap["mergeOfflineParJobsScriptPath"])
repMap["copyMergeScripts"] += (
"cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C .\n"
"rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"])
if anythingToMerge:
# DownloadData is the section which merges output files from parallel jobs
# it uses the file TkAlOfflineJobsMerge.C
repMap["DownloadData"] += replaceByMap(
configTemplates.mergeParallelResults, repMap)
else:
repMap["DownloadData"] = ""
if "OfflineValidation" in comparisonLists:
repMap["extendedValScriptPath"] = os.path.join(
path, "TkAlExtendedOfflineValidation.C")
createExtendedValidationScript(comparisonLists["OfflineValidation"],
repMap["extendedValScriptPath"],
"OfflineValidation")
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
if "TrackSplittingValidation" in comparisonLists:
repMap["trackSplitPlotScriptPath"] = \
os.path.join(path, "TkAlTrackSplitPlot.C")
createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
repMap["trackSplitPlotScriptPath"])
repMap["RunTrackSplitPlot"] = \
replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
if "ZMuMuValidation" in comparisonLists:
repMap["mergeZmumuPlotsScriptPath"] = \
os.path.join(path, "TkAlMergeZmumuPlots.C")
createMergeZmumuPlotsScript(comparisonLists["ZMuMuValidation"],
repMap["mergeZmumuPlotsScriptPath"])
repMap["MergeZmumuPlots"] = \
replaceByMap(configTemplates.mergeZmumuPlotsExecution, repMap)
repMap["CompareAlignments"] = "#run comparisons"
if "OfflineValidation" in comparisonLists:
compareStrings = [
val.getCompareStrings("OfflineValidation")
for val in comparisonLists["OfflineValidation"]
]
compareStringsPlain = [
val.getCompareStrings("OfflineValidation", plain=True)
for val in comparisonLists["OfflineValidation"]
]
repMap.update({
"validationId": "OfflineValidation",
"compareStrings": " , ".join(compareStrings),
"compareStringsPlain": " ".join(compareStringsPlain)
})
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open(filePath, "w")
theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap))
theFile.close()
os.chmod(filePath, 0o755)
return filePath
def createMergeScript( path, validations ):
if(len(validations) == 0):
raise AllInOneError("Cowardly refusing to merge nothing!")
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData":"",
"CompareAlignments":"",
"RunExtendedOfflineValidation":"",
"RunTrackSplitPlot":"",
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"SCRAM_ARCH": os.environ["SCRAM_ARCH"],
"CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
})
comparisonLists = {} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
validationName = validationName.split("Preexisting")[-1]
if validationName in comparisonLists:
comparisonLists[ validationName ].append( validation )
else:
comparisonLists[ validationName ] = [ validation ]
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["haddLoop"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n")
repMap["copyMergeScripts"] = ""
repMap["mergeParallelFilePrefixes"] = ""
anythingToMerge = []
for validationType in comparisonLists:
for validation in comparisonLists[validationType]:
if isinstance(validation, PreexistingValidation) or validation.NJobs == 1:
continue
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
repMap["haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType
repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"])
repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
"xrdcp -f "
+validation.getRepMap()["finalOutputFile"]
+" root://eoscms//eos/cms"
+validation.getRepMap()["finalResultFile"]
+"; fi\n")
repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
"mergeRetCode=${tmpMergeRetCode}; fi\n")
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMap["rmUnmerged"] += " $eos rm "+longName+"\n"
repMap["rmUnmerged"] += ("else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
if "OfflineValidation" in anythingToMerge:
repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"],
repMap["mergeOfflineParJobsScriptPath"] )
repMap["copyMergeScripts"] += ("cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C .\n"
"rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"])
if anythingToMerge:
# DownloadData is the section which merges output files from parallel jobs
# it uses the file TkAlOfflineJobsMerge.C
repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
else:
repMap["DownloadData"] = ""
if "OfflineValidation" in comparisonLists:
repMap["extendedValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
createExtendedValidationScript(comparisonLists["OfflineValidation"],
repMap["extendedValScriptPath"],
"OfflineValidation")
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
if "TrackSplittingValidation" in comparisonLists:
repMap["trackSplitPlotScriptPath"] = \
os.path.join(path, "TkAlTrackSplitPlot.C")
createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
repMap["trackSplitPlotScriptPath"] )
repMap["RunTrackSplitPlot"] = \
replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
repMap["CompareAlignments"] = "#run comparisons"
for validationId in comparisonLists:
compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ]
repMap.update({"validationId": validationId,
"compareStrings": " , ".join(compareStrings),
"compareStringsPlain": " ".join(compareStringsPlain) })
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open( filePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
theFile.close()
os.chmod(filePath,0755)
return filePath
def createMergeScript(path, validations):
if (len(validations) == 0):
raise AllInOneError("Cowardly refusing to merge nothing!")
config = validations[0].config
repMap = config.getGeneral()
repMap.update({
"DownloadData": "",
"CompareAlignments": "",
"RunValidationPlots": "",
"CMSSW_BASE": os.environ["CMSSW_BASE"],
"SCRAM_ARCH": os.environ["SCRAM_ARCH"],
"CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
})
comparisonLists = {
} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationtype = type(validation)
if isinstance(validationtype, PreexistingValidation):
#find the actual validationtype
for parentclass in validationtype.mro():
if not issubclass(parentclass, PreexistingValidation):
validationtype = parentclass
break
key = (validationtype, referenceName)
if key in comparisonLists:
comparisonLists[key].append(validation)
else:
comparisonLists[key] = [validation]
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["doMerge"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = (
"if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n")
repMap["beforeMerge"] = ""
repMap["mergeParallelFilePrefixes"] = ""
anythingToMerge = []
for (validationType,
referencename), validations in comparisonLists.iteritems():
for validation in validations:
if (isinstance(validation, PreexistingValidation)
or validation.NJobs == 1
or not isinstance(validation, ParallelValidation)):
continue
if validationType not in anythingToMerge:
anythingToMerge += [validationType]
repMap[
"doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType
repMap["beforeMerge"] += validationType.doInitMerge()
repMap["doMerge"] += validation.doMerge()
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join("/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMap["rmUnmerged"] += " eos rm " + longName + "\n"
repMap["rmUnmerged"] += (
"else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
if anythingToMerge:
repMap["DownloadData"] += replaceByMap(
configTemplates.mergeParallelResults, repMap)
else:
repMap["DownloadData"] = ""
repMap["RunValidationPlots"] = ""
for (validationType,
referencename), validations in comparisonLists.iteritems():
if issubclass(validationType, ValidationWithPlots):
repMap["RunValidationPlots"] += validationType.doRunPlots(
validations)
repMap["CompareAlignments"] = "#run comparisons"
for (validationType,
referencename), validations in comparisonLists.iteritems():
if issubclass(validationType, ValidationWithComparison):
repMap["CompareAlignments"] += validationType.doComparison(
validations)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open(filePath, "w")
theFile.write(replaceByMap(configTemplates.mergeTemplate, repMap))
theFile.close()
os.chmod(filePath, 0o755)
return filePath
def createMergeScript(path, validations, options):
if (len(validations) == 0):
raise AllInOneError("Cowardly refusing to merge nothing!")
repMap = {}
comparisonLists = {
} # directory of lists containing the validations that are comparable
for validation in validations:
if validation.config.has_section("IOV"):
iov = validation.config.get("IOV", "iov")
validation.defaultReferenceName = iov
for referenceName in validation.filesToCompare:
validationtype = type(validation)
validationName = validation.name
if validation.config.has_section("IOV") and (
referenceName == "Tracker_defaultRange"
or referenceName == "Tracker_autoRange"):
referenceName = iov
if issubclass(validationtype, PreexistingValidation):
validationName = validation.originalValName
#find the actual validationtype
for parentclass in validationtype.mro():
if not issubclass(parentclass, PreexistingValidation):
validationtype = parentclass
break
key = (validationtype, validationName, referenceName)
if key in comparisonLists:
comparisonLists[key].append(validation)
else:
comparisonLists[key] = [validation]
repMap[key] = validation.config.getGeneral()
repMap[key].update({
"DownloadData":
"",
"CompareAlignments":
"",
"RunValidationPlots":
"",
"CMSSW_BASE":
os.environ["CMSSW_BASE"],
"SCRAM_ARCH":
os.environ["SCRAM_ARCH"],
"CMSSW_RELEASE_BASE":
os.environ["CMSSW_RELEASE_BASE"],
})
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap[key]["doMerge"] = "mergeRetCode=0\n"
repMap[key]["rmUnmerged"] = (
"if [[ mergeRetCode -eq 0 ]]; then\n"
" echo -e \\n\"Merging succeeded, removing original files.\"\n"
)
repMap[key]["beforeMerge"] = ""
repMap[key]["mergeParallelFilePrefixes"] = ""
repMap[key]["createResultsDirectory"] = ""
#print("comparisonLists")
#pprint.pprint(comparisonLists)
anythingToMerge = []
for (validationtype, validationName,
referenceName), validations in six.iteritems(comparisonLists):
#pprint.pprint("validations")
#pprint.pprint(validations)
globalDictionaries.plottingOptions = {}
lmap(lambda validation: validation.getRepMap(), validations)
#plotInfo = "plots:offline"
#allPlotInfo = dict(validations[0].config.items(plotInfo))
#repMap[(validationtype, validationName, referenceName)].update(allPlotInfo)
for validation in validations:
validation.getRepMap()
#pprint.pprint("validation in validations")
#pprint.pprint(validation)
#parallel merging
if not (isinstance(validation, PreexistingValidation)
or validation.NJobs == 1
or not isinstance(validation, ParallelValidation)):
if (validationtype, validationName,
referenceName) not in anythingToMerge:
anythingToMerge.append(
(validationtype, validationName, referenceName))
repMap[(
validationtype, validationName, referenceName
)]["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationtype.valType
repMap[(validationtype, validationName, referenceName
)]["beforeMerge"] += validationtype.doInitMerge()
repMap[(validationtype, validationName,
referenceName)]["doMerge"] += validation.doMerge()
for f in validation.getRepMap()["outputFiles"]:
longName = os.path.join(
"/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
validation.getRepMap()["eosdir"], f)
repMap[(validationtype, validationName, referenceName
)]["rmUnmerged"] += " rm " + longName + "\n"
repMap[(validationtype, validationName,
referenceName)]["rmUnmerged"] += (
"else\n"
" echo -e \\n\"WARNING: Merging failed, unmerged"
" files won't be deleted.\\n"
"(Ignore this warning if merging was done earlier)\"\n"
"fi\n")
if anythingToMerge:
repMap[(validationtype, validationName,
referenceName)]["DownloadData"] += replaceByMap(
configTemplates.mergeParallelResults,
repMap[(validationtype, validationName,
referenceName)])
else:
repMap[(validationtype, validationName,
referenceName)]["DownloadData"] = ""
repMap[(validationtype, validationName,
referenceName)]["RunValidationPlots"] = ""
repMap[(validationtype, validationName,
referenceName)]["plottingscriptpath"] = ""
if issubclass(validationtype, ValidationWithPlots):
repMap[(
validationtype, validationName, referenceName
)]["RunValidationPlots"] = validationtype.doRunPlots(validations)
repMap[(validationtype, validationName,
referenceName)]["CompareAlignments"] = "#run comparisons"
if issubclass(validationtype, ValidationWithComparison):
repMap[(
validationtype, validationName, referenceName
)]["CompareAlignments"] += validationtype.doComparison(validations)
#if not merging parallel, add code to create results directory and set merge script name accordingly
if validations[0].config.has_section("IOV"):
repMap[(validationtype, validationName,
referenceName)]["createResultsDirectory"] = replaceByMap(
configTemplates.createResultsDirectoryTemplate,
repMap[(validationtype, validationName,
referenceName)])
filePath = os.path.join(
repMap[(validationtype, validationName,
referenceName)]["scriptsdir"], "TkAlMerge.sh")
else:
repMap[(validationtype, validationName,
referenceName)]["createResultsDirectory"] = replaceByMap(
configTemplates.createResultsDirectoryTemplate,
repMap[(validationtype, validationName,
referenceName)])
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open(filePath, "w")
theFile.write(
replaceByMap(
configTemplates.mergeTemplate,
repMap[(validationtype, validationName, referenceName)]))
theFile.close()
os.chmod(filePath, 0o755)
def createParallelMergeScript( path, validations ):
if( len(validations) == 0 ):
raise AllInOneError, "cowardly refusing to merge nothing!"
repMap = validations[0].getRepMap() #FIXME - not nice this way
repMap.update({
"DownloadData":"",
"CompareAlignments":"",
"RunExtendedOfflineValidation":""
})
comparisonLists = {} # directory of lists containing the validations that are comparable
for validation in validations:
for referenceName in validation.filesToCompare:
validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
if validationName in comparisonLists:
comparisonLists[ validationName ].append( validation )
else:
comparisonLists[ validationName ] = [ validation ]
if "OfflineValidationParallel" in comparisonLists:
repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] )
repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"],
repMap["mergeOfflineParJobsScriptPath"] )
# introduced to merge individual validation outputs separately
# -> avoids problems with merge script
repMap["haddLoop"] = "mergeRetCode=0\n"
repMap["rmUnmerged"] = "if [[ mergeRetCode -eq 0 ]]; then\n"
for validation in comparisonLists["OfflineValidationParallel"]:
repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"])
repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
repMap["haddLoop"] += ("if [[ mergeRetCode -eq 0 ]]; "
"then mergeRetCode=${tmpMergeRetCode}; "
"fi\n")
repMap["haddLoop"] += ("cmsStage -f "
+validation.getRepMap()["outputFile"]
+" "
+validation.getRepMap()["resultFile"]
+"\n")
for f in validation.outputFiles:
longName = os.path.join("/store/caf/user/$USER/",
validation.getRepMap()["eosdir"], f)
repMap["rmUnmerged"] += " cmsRm "+longName+"\n"
repMap["rmUnmerged"] += ("else\n"
" echo \"WARNING: Merging failed, unmerged"
" files won't be deleted.\"\n"
"fi\n")
repMap["RunExtendedOfflineValidation"] = \
replaceByMap(configTemplates.extendedValidationExecution, repMap)
# DownloadData is the section which merges output files from parallel jobs
# it uses the file TkAlOfflineJobsMerge.C
repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
repMap["CompareAlignments"] = "#run comparisons"
for validationId in comparisonLists:
compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
repMap.update({"validationId": validationId,
"compareStrings": " , ".join(compareStrings) })
repMap["CompareAlignments"] += \
replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
filePath = os.path.join(path, "TkAlMerge.sh")
theFile = open( filePath, "w" )
theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
theFile.close()
os.chmod(filePath,0755)
return filePath
