def createConfiguration(self, path ): # self.__compares repMap = self.getRepMap() cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%( self.alignmentToValidate.name) cfgs = {cfgFileName: configTemplates.intoNTuplesTemplate} repMaps = {cfgFileName: repMap} if not self.referenceAlignment == "IDEAL": referenceRepMap = self.getRepMap( self.referenceAlignment ) cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%( self.referenceAlignment.name ) cfgs[cfgFileName] = configTemplates.intoNTuplesTemplate repMaps[cfgFileName] = referenceRepMap cfgSchedule = cfgs.keys() for common in self.__compares: repMap.update({ "levels": self.__compares[common][0], "dbOutput": pythonboolstring(self.__compares[common][1], "dbOutput") }) if self.__compares[common][1].split()[0] == "true": repMap["dbOutputService"] = configTemplates.dbOutputTemplate else: repMap["dbOutputService"] = "" cfgName = replaceByMap(("TkAlCompareCommon.oO[common]Oo.." ".oO[name]Oo._cfg.py"),repMap) cfgs[cfgName] = configTemplates.compareTemplate repMaps[cfgName] = repMap cfgSchedule.append( cfgName ) super(GeometryComparison, self).createConfiguration(cfgs, path, cfgSchedule, repMaps = repMaps)
def __init__(self, valName, alignment, config): super(OfflineValidation, self).__init__(valName, alignment, config) for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals": self.general[name] = pythonboolstring(self.general[name], name) for option in self.deprecateddefaults: if self.general[option]: raise AllInOneError( "The '%s' option has been moved to the [plots:offline] section. Please specify it there." % option) del self.general[option] if self.NJobs > 1 and self.general[ "offlineModuleLevelHistsTransient"] == "True": msg = ("To be able to merge results when running parallel jobs," " set offlineModuleLevelHistsTransient to false.") raise AllInOneError(msg) try: self.NTracks = int(self.general["maxtracks"]) if self.NTracks < 0: raise ValueError except ValueError: raise AllInOneError( "maxtracks has to be a positive integer, or 0 for no limit") if self.NTracks / self.NJobs != float(self.NTracks) / self.NJobs: raise AllInOneError( "maxtracks has to be divisible by parallelJobs")
def __init__(self, valName, alignment, config): super(OfflineValidation, self).__init__(valName, alignment, config) for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals": self.general[name] = pythonboolstring(self.general[name], name) for option in self.deprecateddefaults: if self.general[option]: raise AllInOneError("The '%s' option has been moved to the [plots:offline] section. Please specify it there."%option) del self.general[option] if self.NJobs > 1 and self.general["offlineModuleLevelHistsTransient"] == "True": msg = ("To be able to merge results when running parallel jobs," " set offlineModuleLevelHistsTransient to false.") raise AllInOneError(msg)
def __init__(self, valName, alignment, config): super(OfflineValidation, self).__init__(valName, alignment, config) for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals": self.general[name] = pythonboolstring(self.general[name], name) for option in self.deprecateddefaults: if self.general[option]: raise AllInOneError( "The '%s' option has been moved to the [plots:offline] section. Please specify it there." % option) del self.general[option] if self.NJobs > 1 and self.general[ "offlineModuleLevelHistsTransient"] == "True": msg = ("To be able to merge results when running parallel jobs," " set offlineModuleLevelHistsTransient to false.") raise AllInOneError(msg)
def __init__(self, valName, alignment, config): super(OfflineValidation, self).__init__(valName, alignment, config) for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals": self.general[name] = pythonboolstring(self.general[name], name) for option in self.deprecateddefaults: if self.general[option]: raise AllInOneError("The '%s' option has been moved to the [plots:offline] section. Please specify it there."%option) del self.general[option] if self.NJobs > 1 and self.general["offlineModuleLevelHistsTransient"] == "True": msg = ("To be able to merge results when running parallel jobs," " set offlineModuleLevelHistsTransient to false.") raise AllInOneError(msg) try: self.NTracks = int(self.general["maxtracks"]) if self.NTracks < 0: raise ValueError except ValueError: raise AllInOneError("maxtracks has to be a positive integer, or 0 for no limit") if self.NTracks / self.NJobs != float(self.NTracks) / self.NJobs: raise AllInOneError("maxtracks has to be divisible by parallelJobs")
def createConfiguration(self, path): # self.__compares repMap = self.getRepMap() cfgFileName = "TkAlCompareToNTuple.%s_cfg.py" % ( self.alignmentToValidate.name) cfgs = {cfgFileName: configTemplates.intoNTuplesTemplate} repMaps = {cfgFileName: repMap} if not self.referenceAlignment == "IDEAL": referenceRepMap = self.getRepMap(self.referenceAlignment) cfgFileName = "TkAlCompareToNTuple.%s_cfg.py" % ( self.referenceAlignment.name) cfgs[cfgFileName] = configTemplates.intoNTuplesTemplate repMaps[cfgFileName] = referenceRepMap cfgSchedule = cfgs.keys() for common in self.__compares: repMap.update({ "levels": self.__compares[common][0], "dbOutput": pythonboolstring(self.__compares[common][1], "dbOutput") }) if self.__compares[common][1].split()[0] == "true": repMap["dbOutputService"] = configTemplates.dbOutputTemplate else: repMap["dbOutputService"] = "" cfgName = replaceByMap(("TkAlCompareCommon.oO[common]Oo.." ".oO[name]Oo._cfg.py"), repMap) cfgs[cfgName] = configTemplates.compareTemplate repMaps[cfgName] = repMap cfgSchedule.append(cfgName) super(GeometryComparison, self).createConfiguration(cfgs, path, cfgSchedule, repMaps=repMaps)
def __init__(self, valName, alignment, config): """ This method adds additional items to the `self.general` dictionary which are only needed for validations using datasets. Arguments: - `valName`: String which identifies individual validation instances - `alignment`: `Alignment` instance to validate - `config`: `BetterConfigParser` instance which includes the configuration of the validations """ super(GenericValidationData, self).__init__(valName, alignment, config) # if maxevents is not specified, cannot calculate number of events for # each parallel job, and therefore running only a single job if int(self.general["maxevents"]) < 0 and self.NJobs > 1: msg = ("Maximum number of events (maxevents) not specified: " "cannot use parallel jobs.") raise AllInOneError(msg) if int(self.general["maxevents"]) / self.NJobs != float( self.general["maxevents"]) / self.NJobs: msg = ("maxevents has to be divisible by parallelJobs") raise AllInOneError(msg) tryPredefinedFirst = (not self.jobmode.split(',')[0] == "crab" and self.general["JSON"] == "" and self.general["firstRun"] == "" and self.general["lastRun"] == "" and self.general["begin"] == "" and self.general["end"] == "") if self.general["dataset"] not in globalDictionaries.usedDatasets: globalDictionaries.usedDatasets[self.general["dataset"]] = {} if self.cmssw not in globalDictionaries.usedDatasets[ self.general["dataset"]]: if globalDictionaries.usedDatasets[self.general["dataset"]] != {}: print( "Warning: you use the same dataset '%s' in multiple cmssw releases.\n" "This is allowed, but make sure it's not a mistake" ) % self.general["dataset"] globalDictionaries.usedDatasets[self.general["dataset"]][ self.cmssw] = { False: None, True: None } Bfield = self.general.get("magneticfield", None) if globalDictionaries.usedDatasets[self.general["dataset"]][ self.cmssw][tryPredefinedFirst] is None: dataset = Dataset(self.general["dataset"], tryPredefinedFirst=tryPredefinedFirst, cmssw=self.cmssw, cmsswrelease=self.cmsswreleasebase, magneticfield=Bfield, dasinstance=self.general["dasinstance"]) globalDictionaries.usedDatasets[self.general["dataset"]][ self.cmssw][tryPredefinedFirst] = dataset if tryPredefinedFirst and not dataset.predefined( ): #No point finding the data twice in that case globalDictionaries.usedDatasets[self.general["dataset"]][ self.cmssw][False] = dataset self.dataset = globalDictionaries.usedDatasets[ self.general["dataset"]][self.cmssw][tryPredefinedFirst] self.general["magneticField"] = self.dataset.magneticField() self.general["defaultMagneticField"] = "MagneticField" if self.general["magneticField"] == "unknown": print "Could not get the magnetic field for this dataset." print "Using the default: ", self.general["defaultMagneticField"] self.general["magneticField"] = '.oO[defaultMagneticField]Oo.' if not self.jobmode.split(',')[0] == "crab": try: self.general[ "datasetDefinition"] = self.dataset.datasetSnippet( jsonPath=self.general["JSON"], firstRun=self.general["firstRun"], lastRun=self.general["lastRun"], begin=self.general["begin"], end=self.general["end"], parent=self.needParentFiles) except AllInOneError as e: msg = "In section [%s:%s]: " % (self.valType, self.name) msg += str(e) raise AllInOneError(msg) else: if self.dataset.predefined(): msg = ("For jobmode 'crab' you cannot use predefined datasets " "(in your case: '%s')." % (self.dataset.name())) raise AllInOneError(msg) try: theUpdate = config.getResultingSection( self.valType + ":" + self.name, demandPars=["parallelJobs"]) except AllInOneError as e: msg = str(e)[:-1] + " when using 'jobmode: crab'." raise AllInOneError(msg) self.general.update(theUpdate) if self.general["begin"] or self.general["end"]: (self.general["begin"], self.general["end"], self.general["firstRun"], self.general["lastRun"]) = self.dataset.convertTimeToRun( firstRun=self.general["firstRun"], lastRun=self.general["lastRun"], begin=self.general["begin"], end=self.general["end"], shortTuple=False) if self.general["begin"] == None: self.general["begin"] = "" if self.general["end"] == None: self.general["end"] = "" self.general["firstRun"] = str(self.general["firstRun"]) self.general["lastRun"] = str(self.general["lastRun"]) if ( not self.general["firstRun"] ) and \ ( self.general["end"] or self.general["lastRun"] ): self.general["firstRun"] = str( self.dataset.runList()[0]["run_number"]) if ( not self.general["lastRun"] ) and \ ( self.general["begin"] or self.general["firstRun"] ): self.general["lastRun"] = str( self.dataset.runList()[-1]["run_number"]) if self.general["firstRun"] and self.general["lastRun"]: if int(self.general["firstRun"]) > int( self.general["lastRun"]): msg = ( "The lower time/runrange limit ('begin'/'firstRun') " "chosen is greater than the upper time/runrange limit " "('end'/'lastRun').") raise AllInOneError(msg) self.general["runRange"] = (self.general["firstRun"] + '-' + self.general["lastRun"]) try: self.general[ "datasetDefinition"] = self.dataset.datasetSnippet( jsonPath=self.general["JSON"], firstRun=self.general["firstRun"], lastRun=self.general["lastRun"], begin=self.general["begin"], end=self.general["end"], crab=True) except AllInOneError as e: msg = "In section [%s:%s]: " % (self.valType, self.name) msg += str(e) raise AllInOneError(msg) self.general["usepixelqualityflag"] = pythonboolstring( self.general["usepixelqualityflag"], "usepixelqualityflag")
def __init__(self, valName, alignment, config): """ This method adds additional items to the `self.general` dictionary which are only needed for validations using datasets. Arguments: - `valName`: String which identifies individual validation instances - `alignment`: `Alignment` instance to validate - `config`: `BetterConfigParser` instance which includes the configuration of the validations """ super(GenericValidationData, self).__init__(valName, alignment, config) # if maxevents is not specified, cannot calculate number of events for # each parallel job, and therefore running only a single job if int( self.general["maxevents"] ) < 0 and self.NJobs > 1: msg = ("Maximum number of events (maxevents) not specified: " "cannot use parallel jobs.") raise AllInOneError(msg) if int( self.general["maxevents"] ) / self.NJobs != float( self.general["maxevents"] ) / self.NJobs: msg = ("maxevents has to be divisible by parallelJobs") raise AllInOneError(msg) tryPredefinedFirst = (not self.jobmode.split( ',' )[0] == "crab" and self.general["JSON"] == "" and self.general["firstRun"] == "" and self.general["lastRun"] == "" and self.general["begin"] == "" and self.general["end"] == "") if self.general["dataset"] not in globalDictionaries.usedDatasets: globalDictionaries.usedDatasets[self.general["dataset"]] = {} if self.cmssw not in globalDictionaries.usedDatasets[self.general["dataset"]]: if globalDictionaries.usedDatasets[self.general["dataset"]] != {}: print ("Warning: you use the same dataset '%s' in multiple cmssw releases.\n" "This is allowed, but make sure it's not a mistake") % self.general["dataset"] globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw] = {False: None, True: None} Bfield = self.general.get("magneticfield", None) if globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] is None: dataset = Dataset( self.general["dataset"], tryPredefinedFirst = tryPredefinedFirst, cmssw = self.cmssw, cmsswrelease = self.cmsswreleasebase, magneticfield = Bfield, dasinstance = self.general["dasinstance"]) globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] = dataset if tryPredefinedFirst and not dataset.predefined(): #No point finding the data twice in that case globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][False] = dataset self.dataset = globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] self.general["magneticField"] = self.dataset.magneticField() self.general["defaultMagneticField"] = "MagneticField" if self.general["magneticField"] == "unknown": print "Could not get the magnetic field for this dataset." print "Using the default: ", self.general["defaultMagneticField"] self.general["magneticField"] = '.oO[defaultMagneticField]Oo.' if not self.jobmode.split( ',' )[0] == "crab": try: self.general["datasetDefinition"] = self.dataset.datasetSnippet( jsonPath = self.general["JSON"], firstRun = self.general["firstRun"], lastRun = self.general["lastRun"], begin = self.general["begin"], end = self.general["end"], parent = self.needParentFiles ) except AllInOneError as e: msg = "In section [%s:%s]: "%(self.valType, self.name) msg += str(e) raise AllInOneError(msg) else: if self.dataset.predefined(): msg = ("For jobmode 'crab' you cannot use predefined datasets " "(in your case: '%s')."%( self.dataset.name() )) raise AllInOneError( msg ) try: theUpdate = config.getResultingSection(self.valType+":"+self.name, demandPars = ["parallelJobs"]) except AllInOneError as e: msg = str(e)[:-1]+" when using 'jobmode: crab'." raise AllInOneError(msg) self.general.update(theUpdate) if self.general["begin"] or self.general["end"]: ( self.general["begin"], self.general["end"], self.general["firstRun"], self.general["lastRun"] ) = self.dataset.convertTimeToRun( firstRun = self.general["firstRun"], lastRun = self.general["lastRun"], begin = self.general["begin"], end = self.general["end"], shortTuple = False) if self.general["begin"] == None: self.general["begin"] = "" if self.general["end"] == None: self.general["end"] = "" self.general["firstRun"] = str( self.general["firstRun"] ) self.general["lastRun"] = str( self.general["lastRun"] ) if ( not self.general["firstRun"] ) and \ ( self.general["end"] or self.general["lastRun"] ): self.general["firstRun"] = str( self.dataset.runList()[0]["run_number"]) if ( not self.general["lastRun"] ) and \ ( self.general["begin"] or self.general["firstRun"] ): self.general["lastRun"] = str( self.dataset.runList()[-1]["run_number"]) if self.general["firstRun"] and self.general["lastRun"]: if int(self.general["firstRun"]) > int(self.general["lastRun"]): msg = ( "The lower time/runrange limit ('begin'/'firstRun') " "chosen is greater than the upper time/runrange limit " "('end'/'lastRun').") raise AllInOneError( msg ) self.general["runRange"] = (self.general["firstRun"] + '-' + self.general["lastRun"]) try: self.general["datasetDefinition"] = self.dataset.datasetSnippet( jsonPath = self.general["JSON"], firstRun = self.general["firstRun"], lastRun = self.general["lastRun"], begin = self.general["begin"], end = self.general["end"], crab = True ) except AllInOneError as e: msg = "In section [%s:%s]: "%(self.valType, self.name) msg += str( e ) raise AllInOneError( msg ) self.general["usepixelqualityflag"] = pythonboolstring(self.general["usepixelqualityflag"], "usepixelqualityflag")