def createConfiguration(self, path ):
# self.__compares
repMap = self.getRepMap()
cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%(
self.alignmentToValidate.name)
cfgs = {cfgFileName: configTemplates.intoNTuplesTemplate}
repMaps = {cfgFileName: repMap}
if not self.referenceAlignment == "IDEAL":
referenceRepMap = self.getRepMap( self.referenceAlignment )
cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%(
self.referenceAlignment.name )
cfgs[cfgFileName] = configTemplates.intoNTuplesTemplate
repMaps[cfgFileName] = referenceRepMap
cfgSchedule = cfgs.keys()
for common in self.__compares:
repMap.update({
"levels": self.__compares[common][0],
"dbOutput": pythonboolstring(self.__compares[common][1], "dbOutput")
})
if self.__compares[common][1].split()[0] == "true":
repMap["dbOutputService"] = configTemplates.dbOutputTemplate
else:
repMap["dbOutputService"] = ""
cfgName = replaceByMap(("TkAlCompareCommon.oO[common]Oo.."
".oO[name]Oo._cfg.py"),repMap)
cfgs[cfgName] = configTemplates.compareTemplate
repMaps[cfgName] = repMap
cfgSchedule.append( cfgName )
super(GeometryComparison, self).createConfiguration(cfgs, path, cfgSchedule, repMaps = repMaps)
def __init__(self, valName, alignment, config):
super(OfflineValidation, self).__init__(valName, alignment, config)
for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals":
self.general[name] = pythonboolstring(self.general[name], name)
for option in self.deprecateddefaults:
if self.general[option]:
raise AllInOneError(
"The '%s' option has been moved to the [plots:offline] section. Please specify it there."
% option)
del self.general[option]
if self.NJobs > 1 and self.general[
"offlineModuleLevelHistsTransient"] == "True":
msg = ("To be able to merge results when running parallel jobs,"
" set offlineModuleLevelHistsTransient to false.")
raise AllInOneError(msg)
try:
self.NTracks = int(self.general["maxtracks"])
if self.NTracks < 0: raise ValueError
except ValueError:
raise AllInOneError(
"maxtracks has to be a positive integer, or 0 for no limit")
if self.NTracks / self.NJobs != float(self.NTracks) / self.NJobs:
raise AllInOneError(
"maxtracks has to be divisible by parallelJobs")
def __init__(self, valName, alignment, config):
super(OfflineValidation, self).__init__(valName, alignment, config)
for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals":
self.general[name] = pythonboolstring(self.general[name], name)
for option in self.deprecateddefaults:
if self.general[option]:
raise AllInOneError("The '%s' option has been moved to the [plots:offline] section. Please specify it there."%option)
del self.general[option]
if self.NJobs > 1 and self.general["offlineModuleLevelHistsTransient"] == "True":
msg = ("To be able to merge results when running parallel jobs,"
" set offlineModuleLevelHistsTransient to false.")
raise AllInOneError(msg)
def __init__(self, valName, alignment, config):
super(OfflineValidation, self).__init__(valName, alignment, config)
for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals":
self.general[name] = pythonboolstring(self.general[name], name)
for option in self.deprecateddefaults:
if self.general[option]:
raise AllInOneError(
"The '%s' option has been moved to the [plots:offline] section. Please specify it there."
% option)
del self.general[option]
if self.NJobs > 1 and self.general[
"offlineModuleLevelHistsTransient"] == "True":
msg = ("To be able to merge results when running parallel jobs,"
" set offlineModuleLevelHistsTransient to false.")
raise AllInOneError(msg)
def __init__(self, valName, alignment, config):
super(OfflineValidation, self).__init__(valName, alignment, config)
for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals":
self.general[name] = pythonboolstring(self.general[name], name)
for option in self.deprecateddefaults:
if self.general[option]:
raise AllInOneError("The '%s' option has been moved to the [plots:offline] section. Please specify it there."%option)
del self.general[option]
if self.NJobs > 1 and self.general["offlineModuleLevelHistsTransient"] == "True":
msg = ("To be able to merge results when running parallel jobs,"
" set offlineModuleLevelHistsTransient to false.")
raise AllInOneError(msg)
try:
self.NTracks = int(self.general["maxtracks"])
if self.NTracks < 0: raise ValueError
except ValueError:
raise AllInOneError("maxtracks has to be a positive integer, or 0 for no limit")
if self.NTracks / self.NJobs != float(self.NTracks) / self.NJobs:
raise AllInOneError("maxtracks has to be divisible by parallelJobs")
def createConfiguration(self, path):
# self.__compares
repMap = self.getRepMap()
cfgFileName = "TkAlCompareToNTuple.%s_cfg.py" % (
self.alignmentToValidate.name)
cfgs = {cfgFileName: configTemplates.intoNTuplesTemplate}
repMaps = {cfgFileName: repMap}
if not self.referenceAlignment == "IDEAL":
referenceRepMap = self.getRepMap(self.referenceAlignment)
cfgFileName = "TkAlCompareToNTuple.%s_cfg.py" % (
self.referenceAlignment.name)
cfgs[cfgFileName] = configTemplates.intoNTuplesTemplate
repMaps[cfgFileName] = referenceRepMap
cfgSchedule = cfgs.keys()
for common in self.__compares:
repMap.update({
"levels":
self.__compares[common][0],
"dbOutput":
pythonboolstring(self.__compares[common][1], "dbOutput")
})
if self.__compares[common][1].split()[0] == "true":
repMap["dbOutputService"] = configTemplates.dbOutputTemplate
else:
repMap["dbOutputService"] = ""
cfgName = replaceByMap(("TkAlCompareCommon.oO[common]Oo.."
".oO[name]Oo._cfg.py"), repMap)
cfgs[cfgName] = configTemplates.compareTemplate
repMaps[cfgName] = repMap
cfgSchedule.append(cfgName)
super(GeometryComparison, self).createConfiguration(cfgs,
path,
cfgSchedule,
repMaps=repMaps)
def __init__(self, valName, alignment, config):
"""
This method adds additional items to the `self.general` dictionary
which are only needed for validations using datasets.
Arguments:
- `valName`: String which identifies individual validation instances
- `alignment`: `Alignment` instance to validate
- `config`: `BetterConfigParser` instance which includes the
configuration of the validations
"""
super(GenericValidationData, self).__init__(valName, alignment, config)
# if maxevents is not specified, cannot calculate number of events for
# each parallel job, and therefore running only a single job
if int(self.general["maxevents"]) < 0 and self.NJobs > 1:
msg = ("Maximum number of events (maxevents) not specified: "
"cannot use parallel jobs.")
raise AllInOneError(msg)
if int(self.general["maxevents"]) / self.NJobs != float(
self.general["maxevents"]) / self.NJobs:
msg = ("maxevents has to be divisible by parallelJobs")
raise AllInOneError(msg)
tryPredefinedFirst = (not self.jobmode.split(',')[0] == "crab"
and self.general["JSON"] == ""
and self.general["firstRun"] == ""
and self.general["lastRun"] == ""
and self.general["begin"] == ""
and self.general["end"] == "")
if self.general["dataset"] not in globalDictionaries.usedDatasets:
globalDictionaries.usedDatasets[self.general["dataset"]] = {}
if self.cmssw not in globalDictionaries.usedDatasets[
self.general["dataset"]]:
if globalDictionaries.usedDatasets[self.general["dataset"]] != {}:
print(
"Warning: you use the same dataset '%s' in multiple cmssw releases.\n"
"This is allowed, but make sure it's not a mistake"
) % self.general["dataset"]
globalDictionaries.usedDatasets[self.general["dataset"]][
self.cmssw] = {
False: None,
True: None
}
Bfield = self.general.get("magneticfield", None)
if globalDictionaries.usedDatasets[self.general["dataset"]][
self.cmssw][tryPredefinedFirst] is None:
dataset = Dataset(self.general["dataset"],
tryPredefinedFirst=tryPredefinedFirst,
cmssw=self.cmssw,
cmsswrelease=self.cmsswreleasebase,
magneticfield=Bfield,
dasinstance=self.general["dasinstance"])
globalDictionaries.usedDatasets[self.general["dataset"]][
self.cmssw][tryPredefinedFirst] = dataset
if tryPredefinedFirst and not dataset.predefined(
): #No point finding the data twice in that case
globalDictionaries.usedDatasets[self.general["dataset"]][
self.cmssw][False] = dataset
self.dataset = globalDictionaries.usedDatasets[
self.general["dataset"]][self.cmssw][tryPredefinedFirst]
self.general["magneticField"] = self.dataset.magneticField()
self.general["defaultMagneticField"] = "MagneticField"
if self.general["magneticField"] == "unknown":
print "Could not get the magnetic field for this dataset."
print "Using the default: ", self.general["defaultMagneticField"]
self.general["magneticField"] = '.oO[defaultMagneticField]Oo.'
if not self.jobmode.split(',')[0] == "crab":
try:
self.general[
"datasetDefinition"] = self.dataset.datasetSnippet(
jsonPath=self.general["JSON"],
firstRun=self.general["firstRun"],
lastRun=self.general["lastRun"],
begin=self.general["begin"],
end=self.general["end"],
parent=self.needParentFiles)
except AllInOneError as e:
msg = "In section [%s:%s]: " % (self.valType, self.name)
msg += str(e)
raise AllInOneError(msg)
else:
if self.dataset.predefined():
msg = ("For jobmode 'crab' you cannot use predefined datasets "
"(in your case: '%s')." % (self.dataset.name()))
raise AllInOneError(msg)
try:
theUpdate = config.getResultingSection(
self.valType + ":" + self.name,
demandPars=["parallelJobs"])
except AllInOneError as e:
msg = str(e)[:-1] + " when using 'jobmode: crab'."
raise AllInOneError(msg)
self.general.update(theUpdate)
if self.general["begin"] or self.general["end"]:
(self.general["begin"], self.general["end"],
self.general["firstRun"],
self.general["lastRun"]) = self.dataset.convertTimeToRun(
firstRun=self.general["firstRun"],
lastRun=self.general["lastRun"],
begin=self.general["begin"],
end=self.general["end"],
shortTuple=False)
if self.general["begin"] == None:
self.general["begin"] = ""
if self.general["end"] == None:
self.general["end"] = ""
self.general["firstRun"] = str(self.general["firstRun"])
self.general["lastRun"] = str(self.general["lastRun"])
if ( not self.general["firstRun"] ) and \
( self.general["end"] or self.general["lastRun"] ):
self.general["firstRun"] = str(
self.dataset.runList()[0]["run_number"])
if ( not self.general["lastRun"] ) and \
( self.general["begin"] or self.general["firstRun"] ):
self.general["lastRun"] = str(
self.dataset.runList()[-1]["run_number"])
if self.general["firstRun"] and self.general["lastRun"]:
if int(self.general["firstRun"]) > int(
self.general["lastRun"]):
msg = (
"The lower time/runrange limit ('begin'/'firstRun') "
"chosen is greater than the upper time/runrange limit "
"('end'/'lastRun').")
raise AllInOneError(msg)
self.general["runRange"] = (self.general["firstRun"] + '-' +
self.general["lastRun"])
try:
self.general[
"datasetDefinition"] = self.dataset.datasetSnippet(
jsonPath=self.general["JSON"],
firstRun=self.general["firstRun"],
lastRun=self.general["lastRun"],
begin=self.general["begin"],
end=self.general["end"],
crab=True)
except AllInOneError as e:
msg = "In section [%s:%s]: " % (self.valType, self.name)
msg += str(e)
raise AllInOneError(msg)
self.general["usepixelqualityflag"] = pythonboolstring(
self.general["usepixelqualityflag"], "usepixelqualityflag")
def __init__(self, valName, alignment, config):
"""
This method adds additional items to the `self.general` dictionary
which are only needed for validations using datasets.
Arguments:
- `valName`: String which identifies individual validation instances
- `alignment`: `Alignment` instance to validate
- `config`: `BetterConfigParser` instance which includes the
configuration of the validations
"""
super(GenericValidationData, self).__init__(valName, alignment, config)
# if maxevents is not specified, cannot calculate number of events for
# each parallel job, and therefore running only a single job
if int( self.general["maxevents"] ) < 0 and self.NJobs > 1:
msg = ("Maximum number of events (maxevents) not specified: "
"cannot use parallel jobs.")
raise AllInOneError(msg)
if int( self.general["maxevents"] ) / self.NJobs != float( self.general["maxevents"] ) / self.NJobs:
msg = ("maxevents has to be divisible by parallelJobs")
raise AllInOneError(msg)
tryPredefinedFirst = (not self.jobmode.split( ',' )[0] == "crab" and self.general["JSON"] == ""
and self.general["firstRun"] == "" and self.general["lastRun"] == ""
and self.general["begin"] == "" and self.general["end"] == "")
if self.general["dataset"] not in globalDictionaries.usedDatasets:
globalDictionaries.usedDatasets[self.general["dataset"]] = {}
if self.cmssw not in globalDictionaries.usedDatasets[self.general["dataset"]]:
if globalDictionaries.usedDatasets[self.general["dataset"]] != {}:
print ("Warning: you use the same dataset '%s' in multiple cmssw releases.\n"
"This is allowed, but make sure it's not a mistake") % self.general["dataset"]
globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw] = {False: None, True: None}
Bfield = self.general.get("magneticfield", None)
if globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] is None:
dataset = Dataset(
self.general["dataset"], tryPredefinedFirst = tryPredefinedFirst,
cmssw = self.cmssw, cmsswrelease = self.cmsswreleasebase, magneticfield = Bfield,
dasinstance = self.general["dasinstance"])
globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] = dataset
if tryPredefinedFirst and not dataset.predefined(): #No point finding the data twice in that case
globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][False] = dataset
self.dataset = globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst]
self.general["magneticField"] = self.dataset.magneticField()
self.general["defaultMagneticField"] = "MagneticField"
if self.general["magneticField"] == "unknown":
print "Could not get the magnetic field for this dataset."
print "Using the default: ", self.general["defaultMagneticField"]
self.general["magneticField"] = '.oO[defaultMagneticField]Oo.'
if not self.jobmode.split( ',' )[0] == "crab":
try:
self.general["datasetDefinition"] = self.dataset.datasetSnippet(
jsonPath = self.general["JSON"],
firstRun = self.general["firstRun"],
lastRun = self.general["lastRun"],
begin = self.general["begin"],
end = self.general["end"],
parent = self.needParentFiles )
except AllInOneError as e:
msg = "In section [%s:%s]: "%(self.valType, self.name)
msg += str(e)
raise AllInOneError(msg)
else:
if self.dataset.predefined():
msg = ("For jobmode 'crab' you cannot use predefined datasets "
"(in your case: '%s')."%( self.dataset.name() ))
raise AllInOneError( msg )
try:
theUpdate = config.getResultingSection(self.valType+":"+self.name,
demandPars = ["parallelJobs"])
except AllInOneError as e:
msg = str(e)[:-1]+" when using 'jobmode: crab'."
raise AllInOneError(msg)
self.general.update(theUpdate)
if self.general["begin"] or self.general["end"]:
( self.general["begin"],
self.general["end"],
self.general["firstRun"],
self.general["lastRun"] ) = self.dataset.convertTimeToRun(
firstRun = self.general["firstRun"],
lastRun = self.general["lastRun"],
begin = self.general["begin"],
end = self.general["end"],
shortTuple = False)
if self.general["begin"] == None:
self.general["begin"] = ""
if self.general["end"] == None:
self.general["end"] = ""
self.general["firstRun"] = str( self.general["firstRun"] )
self.general["lastRun"] = str( self.general["lastRun"] )
if ( not self.general["firstRun"] ) and \
( self.general["end"] or self.general["lastRun"] ):
self.general["firstRun"] = str(
self.dataset.runList()[0]["run_number"])
if ( not self.general["lastRun"] ) and \
( self.general["begin"] or self.general["firstRun"] ):
self.general["lastRun"] = str(
self.dataset.runList()[-1]["run_number"])
if self.general["firstRun"] and self.general["lastRun"]:
if int(self.general["firstRun"]) > int(self.general["lastRun"]):
msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
"chosen is greater than the upper time/runrange limit "
"('end'/'lastRun').")
raise AllInOneError( msg )
self.general["runRange"] = (self.general["firstRun"]
+ '-' + self.general["lastRun"])
try:
self.general["datasetDefinition"] = self.dataset.datasetSnippet(
jsonPath = self.general["JSON"],
firstRun = self.general["firstRun"],
lastRun = self.general["lastRun"],
begin = self.general["begin"],
end = self.general["end"],
crab = True )
except AllInOneError as e:
msg = "In section [%s:%s]: "%(self.valType, self.name)
msg += str( e )
raise AllInOneError( msg )
self.general["usepixelqualityflag"] = pythonboolstring(self.general["usepixelqualityflag"], "usepixelqualityflag")