def dump_cff(self, outName=None, jsonPath=None, begin=None, end=None, firstRun=None, lastRun=None): if outName == None: outName = "Dataset" packageName = os.path.join("Alignment", "OfflineValidation") if not os.path.exists( os.path.join(os.environ["CMSSW_BASE"], "src", packageName)): msg = ("You try to store the predefined dataset'%s'.\n" "For that you need to check out the package '%s' to your " "private relase area in\n" % (outName, packageName) + os.environ["CMSSW_BASE"]) raise AllInOneError(msg) theMap = { "process": "", "tab": "", "nEvents": str(-1), "importCms": "import FWCore.ParameterSet.Config as cms\n" } dataset_cff = self.__createSnippet(jsonPath=jsonPath, begin=begin, end=end, firstRun=firstRun, lastRun=lastRun, repMap=theMap) filePath = os.path.join(os.environ["CMSSW_BASE"], "src", packageName, "python", outName + "_cff.py") if os.path.exists(filePath): existMsg = "The predefined dataset '%s' already exists.\n" % ( outName) askString = "Do you want to overwrite it? [y/n]\n" inputQuery = existMsg + askString while True: userInput = raw_input(inputQuery).lower() if userInput == "y": break elif userInput == "n": return else: inputQuery = askString print( "The predefined dataset '%s' will be stored in the file\n" % (outName) + filePath + "\nFor future use you have to do 'scram b'.") print theFile = open(filePath, "w") theFile.write(dataset_cff) theFile.close() return
def __getMagneticFieldForRun( self, run = -1, tolerance = 0.5 ): """For MC, this returns the same as the previous function. For data, it gets the magnetic field from the runs. This is important for deciding which template to use for offlinevalidation """ if self.__dataType == "mc" and self.__magneticField == "MagneticField": return 3.8 #For 3.8T MC the default MagneticField is used if "T" in self.__magneticField: Bfield = self.__magneticField.split("T")[0].replace("MagneticField_","") try: return float(Bfield) / 10.0 #e.g. 38T and 38T_PostLS1 both return 3.8 except ValueError: pass if self.__predefined: with open(self.__filename) as f: Bfield = None for line in f.readlines(): if line.startswith("#magnetic field: ") and "," in line: if Bfield is not None: raise AllInOneError(self.__filename + " has multiple 'magnetic field' lines.") return float(line.replace("#magnetic field: ", "").split(",")[1].split("#")[0].strip()) if run > 0: dasQuery = ('run = %s'%run) #for data data = self.__getData(dasQuery) try: return self.__findInJson(data, ["run","bfield"]) except KeyError: return "unknown Can't get the magnetic field for run %s from DAS" % run #run < 0 - find B field for the first and last runs, and make sure they're compatible # (to within tolerance) #NOT FOOLPROOF! The magnetic field might go up and then down, or vice versa if self.__firstusedrun is None or self.__lastusedrun is None: return "unknown Can't get the exact magnetic field for the dataset until data has been retrieved from DAS." firstrunB = self.__getMagneticFieldForRun(self.__firstusedrun) lastrunB = self.__getMagneticFieldForRun(self.__lastusedrun) try: if abs(firstrunB - lastrunB) <= tolerance: return .5*(firstrunB + lastrunB) print firstrunB, lastrunB, tolerance return ("unknown The beginning and end of your run range for %s\n" "have different magnetic fields (%s, %s)!\n" "Try limiting the run range using firstRun, lastRun, begin, end, or JSON,\n" "or increasing the tolerance (in dataset.py) from %s.") % (self.__name, firstrunB, lastrunB, tolerance) except TypeError: try: if "unknown" in firstrunB: return firstrunB else: return lastrunB except TypeError: return lastrunB
def __getData(self, dasQuery, dasLimit=0): dasData = das_client.get_data('https://cmsweb.cern.ch', dasQuery, 0, dasLimit, False) if isinstance(dasData, str): jsondict = json.loads(dasData) else: jsondict = dasData # Check, if the DAS query fails if jsondict["status"] != 'ok': msg = "Status not 'ok', but:", jsondict["status"] raise AllInOneError(msg) return jsondict["data"]
def __init__(self, valName, config): self.general = config.getGeneral() self.name = self.general["name"] = valName self.config = config theUpdate = config.getResultingSection("preexisting" + self.valType + ":" + self.name, defaultDict=self.defaults, demandPars=self.mandatories) self.general.update(theUpdate) self.title = self.general["title"] if "|" in self.title or "," in self.title or '"' in self.title: msg = "The characters '|', '\"', and ',' cannot be used in the alignment title!" raise AllInOneError(msg) self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy") self.jobid = self.general["jobid"] if self.jobid: try: #make sure it's actually a valid jobid output = getCommandOutput2("bjobs %(jobid)s 2>&1" % self.general) if "is not found" in output: raise RuntimeError except RuntimeError: raise AllInOneError( "%s is not a valid jobid.\nMaybe it finished already?" % self.jobid) knownOpts = set( self.defaults.keys()) | self.mandatories | self.optionals ignoreOpts = [] config.checkInput("preexisting" + self.valType + ":" + self.name, knownSimpleOptions=knownOpts, ignoreOptions=ignoreOpts) self.jobmode = None try: #initialize plotting options for this validation type result = PlottingOptions(self.config, self.valType) except KeyError: pass
def convertTimeToRun(self, begin=None, end=None, firstRun=None, lastRun=None, shortTuple=True): if (begin and firstRun) or (end and lastRun): msg = ( "The Usage of " + "'begin' & 'firstRun' " * int(bool(begin and firstRun)) + "and " * int(bool( (begin and firstRun) and (end and lastRun))) + "'end' & 'lastRun' " * int(bool(end and lastRun)) + "is ambigous.") raise AllInOneError(msg) runList = [run["run_number"] for run in self.__getRunList()] runTimeList = [run["creation_time"] for run in self.__getRunList()] if begin: try: runIndex = self.__find_ge(runTimeList, begin) except ValueError: msg = ("Your 'begin' is after the creation time of the last " "run in the dataset\n'%s'" % (self.__name)) raise AllInOneError(msg) firstRun = runList[runIndex] begin = None if end: try: runIndex = self.__find_lt(runTimeList, end) except ValueError: msg = ("Your 'end' is before the creation time of the first " "run in the dataset\n'%s'" % (self.__name)) raise AllInOneError(msg) lastRun = runList[runIndex] end = None if shortTuple: return firstRun, lastRun else: return begin, end, firstRun, lastRun
def createConfiguration(self, path, configBaseName = "TkAlOfflineValidation"): # if offline validation uses N parallel jobs, we create here N cfg files numberParallelJobs = int( self.general["parallelJobs"] ) # limit maximum number of parallel jobs to 40 # (each output file is approximately 20MB) maximumNumberJobs = 40 if numberParallelJobs > maximumNumberJobs: msg = ("Maximum allowed number of parallel jobs " +str(maximumNumberJobs)+" exceeded!!!") raise AllInOneError(msg) # if maxevents is not specified, cannot calculate number of events for # each parallel job, and therefore running only a single job if int( self.general["maxevents"] ) == -1: msg = ("Maximum number of events (maxevents) not specified: " "cannot use parallel jobs in offline validation") raise AllInOneError(msg) if numberParallelJobs > 1: if self.general["offlineModuleLevelHistsTransient"] == "True": msg = ("To be able to merge results when running parallel jobs," " set offlineModuleLevelHistsTransient to false.") raise AllInOneError(msg) for index in range(numberParallelJobs): cfgName = "%s.%s.%s_%s_cfg.py"%(configBaseName, self.name, self.alignmentToValidate.name, str(index)) repMap = self.getRepMap() # in this parallel job, skip index*(maxEvents/nJobs) events from # the beginning # (first index is zero, so no skipping for a single job) # and use _index_ in the name of the output file repMap.update({"nIndex": str(index)}) # Create the result file directly to datadir since should not use /tmp/ # see https://cern.service-now.com/service-portal/article.do?n=KB0000484 repMap.update({"outputFile": self.outputFiles[index]}) repMap["outputFile"] = os.path.expandvars( repMap["outputFile"] ) cfgs = {cfgName:replaceByMap(configTemplates.offlineParallelTemplate, repMap)} self.filesToCompare[GenericValidationData.defaultReferenceName] = repMap["resultFile"] GenericValidationData.createConfiguration(self, cfgs, path)
def __init__(self, valName, alignment, referenceAlignment, config, copyImages=True, randomWorkdirPart=None): """ Constructor of the GeometryComparison class. Arguments: - `valName`: String which identifies individual validation instances - `alignment`: `Alignment` instance to validate - `referenceAlignment`: `Alignment` instance which is compared with `alignment` - `config`: `BetterConfigParser` instance which includes the configuration of the validations - `copyImages`: Boolean which indicates whether png- and pdf-files should be copied back from the batch farm - `randomWorkDirPart`: If this option is ommitted a random number is generated to create unique path names for the individual validation instances. """ defaults = { "3DSubdetector1": "1", "3DSubdetector2": "2", "3DTranslationalScaleFactor": "50" } mandatories = ["levels", "dbOutput"] GenericValidation.__init__(self, valName, alignment, config, "compare", addDefaults=defaults, addMandatories=mandatories) if not randomWorkdirPart == None: self.randomWorkdirPart = randomWorkdirPart self.referenceAlignment = referenceAlignment referenceName = "IDEAL" if not self.referenceAlignment == "IDEAL": referenceName = self.referenceAlignment.name allCompares = config.getCompares() self.__compares = {} if valName in allCompares: self.__compares[valName] = allCompares[valName] else: msg = ("Could not find compare section '%s' in '%s'" % (valName, allCompares)) raise AllInOneError(msg) self.copyImages = copyImages
def parsestyle(style): try: #simplest case: it's an int return int(style) except ValueError: pass try: #kStar, kDot, ... style = str(getattr(ROOT,style)) return int(style) except (AttributeError, ValueError): pass raise AllInOneError("style has to be an integer or a ROOT constant (kDashed, kStar, ...)!")
def __init__(self, valName, config, valType, addDefaults = {}, addMandatories=[]): self.name = valName self.general = config.getGeneral() self.config = config self.filesToCompare = {} defaults = {"title": self.name, "jobid": "", "subdetector": "BPIX"} defaults.update(addDefaults) mandatories = ["file", "color", "style"] mandatories += addMandatories theUpdate = config.getResultingSection("preexisting"+valType+":"+self.name, defaultDict = defaults, demandPars = mandatories) self.general.update(theUpdate) self.title = self.general["title"] if "|" in self.title or "," in self.title or '"' in self.title: msg = "The characters '|', '\"', and ',' cannot be used in the alignment title!" raise AllInOneError(msg) self.jobid = self.general["jobid"] if self.jobid: try: #make sure it's actually a valid jobid output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general) if "is not found" in output: raise RuntimeError except RuntimeError: raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid) self.filesToCompare[GenericValidationData.defaultReferenceName] = \ self.general["file"] knownOpts = defaults.keys()+mandatories ignoreOpts = [] config.checkInput("preexisting"+valType+":"+self.name, knownSimpleOptions = knownOpts, ignoreOptions = ignoreOpts) self.jobmode = None
def __updateDict(self, dictionary, section): result = dictionary try: for option in self.options(section): result[option] = self.get(section, option) if "local" + section.title() in self.sections(): for option in self.options("local" + section.title()): result[option] = self.get("local" + section.title(), option) except ConfigParser.NoSectionError, section: msg = ("%s in configuration files. This section is mandatory." % (str(section).replace(":", "", 1))) raise AllInOneError(msg)
def datasetSnippet( self, jsonPath = None, begin = None, end = None, firstRun = None, lastRun = None, crab = False, parent = False ): if self.__predefined and parent: with open(self.__filename) as f: if "secFiles.extend" not in f.read(): msg = ("The predefined dataset '%s' does not contain secondary files, " "which your validation requires!") % self.__name if self.__official: self.__name = self.__origName self.__predefined = False print msg print ("Retreiving the files from DAS. You will be asked if you want " "to overwrite the old dataset.\n" "It will still be compatible with validations that don't need secondary files.") else: raise AllInOneError(msg) if self.__predefined: snippet = ("process.load(\"Alignment.OfflineValidation.%s_cff\")\n" "process.maxEvents = cms.untracked.PSet(\n" " input = cms.untracked.int32(.oO[nEvents]Oo. / .oO[parallelJobs]Oo.)\n" ")\n" "process.source.skipEvents=cms.untracked.uint32(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.)" %(self.__name)) if not parent: with open(self.__filename) as f: if "secFiles.extend" in f.read(): snippet += "\nprocess.source.secondaryFileNames = cms.untracked.vstring()" return snippet theMap = { "process": "process.", "tab": " " * len( "process." ), "nEvents": ".oO[nEvents]Oo. / .oO[parallelJobs]Oo.", "skipEventsString": "process.source.skipEvents=cms.untracked.uint32(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.)\n", "importCms": "", "header": "" } datasetSnippet = self.__createSnippet( jsonPath = jsonPath, begin = begin, end = end, firstRun = firstRun, lastRun = lastRun, repMap = theMap, crab = crab, parent = parent ) if jsonPath == "" and begin == "" and end == "" and firstRun == "" and lastRun == "": try: self.dump_cff(parent = parent) except AllInOneError, e: print "Can't store the dataset as a cff:" print e print "This may be inconvenient in the future, but will not cause a problem for this validation."
def addIndex(filename, njobs, index = None): if index is None: return [addIndex(filename, njobs, i) for i in range(njobs)] if njobs == 1: return filename fileExtension = None for extension in fileExtensions: if filename.endswith(extension): fileExtension = extension if fileExtension is None: raise AllInOneError(fileName + " does not end with any of the extensions " + str(fileExtensions)) return replacelast(filename, fileExtension, "_" + str(index) + fileExtension)
def __init__(self, config, addDefaults = {}, addMandatories=[], addneedpackages=[]): defaults = { "outliercut": "-1.0", "subdetector": "none", } defaults.update(addDefaults) mandatories = [] mandatories += addMandatories needpackages = ["Alignment/CommonAlignmentProducer"] needpackages += addneedpackages BasePlottingOptions.__init__(self, config, "split", defaults, mandatories, needpackages) validsubdets = self.validsubdets() if self.general["subdetector"] not in validsubdets: raise AllInOneError("'%s' is not a valid subdetector!\n" % self.general["subdetector"] + "The options are: " + ", ".join(validsubdets))
def createConfiguration(self, path): cfgName = "%s.%s.%s_cfg.py"%( self.configBaseName, self.name, self.alignmentToValidate.name ) repMap = self.getRepMap() if self.NJobs > 1 and self.general["offlineModuleLevelHistsTransient"] == "True": msg = ("To be able to merge results when running parallel jobs," " set offlineModuleLevelHistsTransient to false.") raise AllInOneError(msg) templateToUse = configTemplates.offlineTemplate if self.AutoAlternates: if "Cosmics" in self.general["trackcollection"]: Bfield = self.dataset.magneticFieldForRun() if Bfield > 3.3 and Bfield < 4.3: #Should never be 4.3, but this covers strings, which always compare bigger than ints templateToUse = configTemplates.CosmicsOfflineValidation print ("B field for %s = %sT. Using the template for cosmics at 3.8T.\n" "To override this behavior, specify AutoAlternates = false in the [alternateTemplates] section") % (self.dataset.name(), Bfield) elif Bfield < 0.5: templateToUse = configTemplates.CosmicsAt0TOfflineValidation print ("B field for %s = %sT. Using the template for cosmics at 0T.\n" "To override this behavior, specify AutoAlternates = false in the [alternateTemplates] section") % (self.dataset.name(), Bfield) else: try: if "unknown " in Bfield: msg = Bfield.replace("unknown ","",1) elif "Bfield" is "unknown": msg = "Can't get the B field for %s." % self.dataset.name() except TypeError: msg = "B field for %s = %sT. This is not that close to 0T or 3.8T." % (self.dataset.name(), Bfield) raise AllInOneError(msg + "\n" "To use this data, turn off the automatic alternates using AutoAlternates = false\n" "in the [alternateTemplates] section, and choose the alternate template yourself.") cfgs = {cfgName: templateToUse} self.filesToCompare[ GenericValidationData.defaultReferenceName ] = repMap["finalResultFile"] return GenericValidationData.createConfiguration(self, cfgs, path, repMap = repMap)
def __init__(self, datasetName, dasLimit=0): self.__name = datasetName # check, if dataset name matches CMS dataset naming scheme if re.match(r'/.+/.+/.+', self.__name): self.__dataType = self.__getDataType() self.__predefined = False else: fileName = self.__name + "_cff.py" searchPath1 = os.path.join(os.environ["CMSSW_BASE"], "python", "Alignment", "OfflineValidation", fileName) searchPath2 = os.path.join(os.environ["CMSSW_BASE"], "src", "Alignment", "OfflineValidation", "python", fileName) searchPath3 = os.path.join(os.environ["CMSSW_RELEASE_BASE"], "python", "Alignment", "OfflineValidation", fileName) if os.path.exists(searchPath1): pass elif os.path.exists(searchPath2): msg = ("The predefined dataset '%s' does exist in '%s', but " "you need to run 'scram b' first." % (self.__name, searchPath2)) raise AllInOneError(msg) elif os.path.exists(searchPath3): pass else: msg = ("The predefined dataset '%s' does not exist. Please " "create it first or check for typos." % (self.__name)) raise AllInOneError(msg) self.__dataType = "unknown" self.__predefined = True self.__dasLimit = dasLimit self.__fileList = None self.__fileInfoList = None self.__runList = None
def __init__(self, valName, alignment, config): super(PrimaryVertexValidation, self).__init__(valName, alignment, config) for name in "doBPix", "doFPix": self.general[name] = pythonboolstring(self.general[name], name) if self.general["pvvalidationreference"].startswith("/store"): self.general[ "pvvalidationreference"] = "root://eoscms//eos/cms" + self.general[ "pvvalidationreference"] if self.NJobs > 1: raise AllInOneError( "Parallel jobs not implemented for the PrimaryVertex validation!\n" "Please set parallelJobs = 1.")
def __init__(self, valName, alignment, config, configBaseName="TkAlZMuMuValidation", scriptBaseName="TkAlZMuMuValidation", crabCfgBaseName="TkAlZMuMuValidation", resultBaseName="ZMuMuValidation", outputBaseName="ZMuMuValidation"): defaults = { "zmumureference": ("/store/caf/user/emiglior/Alignment/TkAlDiMuonValidation/Reference/BiasCheck_DYToMuMu_Summer12_TkAlZMuMu_IDEAL.root" ), "resonance": "Z", "switchONfit": "false", "rebinphi": "4", "rebinetadiff": "2", "rebineta": "2", "rebinpt": "8", } mandatories = ["etamaxneg", "etaminneg", "etamaxpos", "etaminpos"] self.configBaseName = configBaseName self.scriptBaseName = scriptBaseName self.crabCfgBaseName = crabCfgBaseName self.resultBaseName = resultBaseName self.outputBaseName = outputBaseName self.needParentFiles = False GenericValidationData.__init__(self, valName, alignment, config, "zmumu", addDefaults=defaults, addMandatories=mandatories) if self.general["zmumureference"].startswith("/store"): self.general[ "zmumureference"] = "root://eoscms//eos/cms" + self.general[ "zmumureference"] if self.NJobs > 1: raise AllInOneError( "Parallel jobs not implemented for the Z->mumu validation!\n" "Please set parallelJobs = 1.")
def __init__(self, valName, alignment, config, addDefaults={}, addMandatories=[], configBaseName="TkAlOfflineValidation", scriptBaseName="TkAlOfflineValidation", crabCfgBaseName="TkAlOfflineValidation", resultBaseName="AlignmentValidation", outputBaseName="AlignmentValidation"): defaults = { "offlineModuleLevelHistsTransient": "False", "offlineModuleLevelProfiles": "True", "stripYResiduals": "False", } deprecateddefaults = { "DMRMethod": "", "DMRMinimum": "", "DMROptions": "", "OfflineTreeBaseDir": "", "SurfaceShapes": "", } mandatories = ["trackcollection"] defaults.update(deprecateddefaults) defaults.update(addDefaults) mandatories += addMandatories self.configBaseName = configBaseName self.scriptBaseName = scriptBaseName self.crabCfgBaseName = crabCfgBaseName self.resultBaseName = resultBaseName self.outputBaseName = outputBaseName self.needParentFiles = False GenericValidationData.__init__(self, valName, alignment, config, "offline", addDefaults=defaults, addMandatories=mandatories) for option in deprecateddefaults: if self.general[option]: raise AllInOneError( "The '%s' option has been moved to the [plots:offline] section. Please specify it there." % option) del self.general[option]
def __init__(self, valName, alignment, config, configBaseName="TkAlPrimaryVertexValidation", scriptBaseName="TkAlPrimaryVertexValidation", crabCfgBaseName="TkAlPrimaryVertexValidation", resultBaseName="PrimaryVertexValidation", outputBaseName="PrimaryVertexValidation"): defaults = { "pvvalidationreference": ("/store/caf/user/musich/Alignment/TkAlPrimaryVertexValidation/Reference/PrimaryVertexValidation_test_pvvalidation_mc_design_mc_48bins.root" ), "ttrhbuilder": "WithAngleAndTemplate", "doBPix": "True", "doFPix": "True" } mandatories = [ "isda", "ismc", "runboundary", "trackcollection", "vertexcollection", "lumilist", "ptCut", "etaCut", "runControl", "numberOfBins" ] self.configBaseName = configBaseName self.scriptBaseName = scriptBaseName self.crabCfgBaseName = crabCfgBaseName self.resultBaseName = resultBaseName self.outputBaseName = outputBaseName self.needParentFiles = False GenericValidationData.__init__(self, valName, alignment, config, "primaryvertex", addDefaults=defaults, addMandatories=mandatories) if self.general["pvvalidationreference"].startswith("/store"): self.general[ "pvvalidationreference"] = "root://eoscms//eos/cms" + self.general[ "pvvalidationreference"] if self.NJobs > 1: raise AllInOneError( "Parallel jobs not implemented for the PrimaryVertex validation!\n" "Please set parallelJobs = 1.")
def cosmics0T(self): if "Cosmics" not in self.general["trackcollection"]: return False Bfield = self.dataset.magneticFieldForRun() if Bfield < 0.5: return True if isinstance(Bfield, str): if "unknown " in Bfield: msg = Bfield.replace("unknown ", "", 1) elif Bfield == "unknown": msg = "Can't get the B field for %s." % self.dataset.name() else: msg = "B field = {}???".format(Bfield) raise AllInOneError( msg + "\n" "To use this dataset, specify magneticfield = [value] in your .ini config file." ) return False
def __init__(self, valName, config, addDefaults = {}, addMandatories=[]): defaults = {} deprecateddefaults = { "DMRMethod":"", "DMRMinimum":"", "DMROptions":"", "OfflineTreeBaseDir":"", "SurfaceShapes":"" } defaults.update(deprecateddefaults) defaults.update(addDefaults) PreexistingValidation.__init__(self, valName, config, "offline", defaults, addMandatories) for option in deprecateddefaults: if self.general[option]: raise AllInOneError("The '%s' option has been moved to the [plots:offline] section. Please specify it there."%option)
def __init__(self, valName, alignment, config, configBaseName="TkAlOfflineValidationDQM"): OfflineValidation.__init__(self, valName, alignment, config, configBaseName=configBaseName) if not config.has_section("DQM"): msg = "You need to have a DQM section in your configfile!" raise AllInOneError(msg) self.__PrimaryDataset = config.get("DQM", "primaryDataset") self.__firstRun = int(config.get("DQM", "firstRun")) self.__lastRun = int(config.get("DQM", "lastRun"))
def getAlignments(self): alignments = [] for section in self.sections(): if "alignment:" in section: alignments.append( Alignment(section.split("alignment:")[1], self)) names_after_cleaning = [alignment.name for alignment in alignments] duplicates = [ name for name, count in collections.Counter( names_after_cleaning).items() if count > 1 ] if len(duplicates) > 0: msg = "Duplicate alignment names after removing invalid characters: " msg += ", ".join(duplicates) + "\n" msg += "Please rename the alignments to avoid name clashes." raise AllInOneError(msg) return alignments
def getRepMap(self, alignment = None): repMap = OfflineValidation.getRepMap(self, alignment) repMap.update({ "workdir": os.path.expandvars(repMap["workdir"]), "offlineValidationMode": "Dqm", "offlineValidationFileOutput": configTemplates.offlineDqmFileOutputTemplate, "workflow": ("/%s/TkAl%s-.oO[alignmentName]Oo._R%09i_R%09i_" "ValSkim-v1/ALCARECO" %(self.__PrimaryDataset, datetime.datetime.now().strftime("%y"), self.__firstRun, self.__lastRun)), "firstRunNumber": "%i"% self.__firstRun }) if "__" in repMap["workflow"]: msg = ("the DQM workflow specefication must not contain '__'. " "it is: %s"%repMap["workflow"]) raise AllInOneError(msg) return repMap
def __init__(self, valName, alignment, referenceAlignment, config, copyImages=True, randomWorkdirPart=None): """ Constructor of the GeometryComparison class. Arguments: - `valName`: String which identifies individual validation instances - `alignment`: `Alignment` instance to validate - `referenceAlignment`: `Alignment` instance which is compared with `alignment` - `config`: `BetterConfigParser` instance which includes the configuration of the validations - `copyImages`: Boolean which indicates whether png- and pdf-files should be copied back from the batch farm - `randomWorkDirPart`: If this option is ommitted a random number is generated to create unique path names for the individual validation instances. """ GenericValidation.__init__(self, valName, alignment, config) if not randomWorkdirPart == None: self.randomWorkdirPart = randomWorkdirPart self.referenceAlignment = referenceAlignment try: # try to override 'jobmode' from [general] section self.jobmode = config.get("compare:" + self.name, "jobmode") except ConfigParser.NoOptionError: pass referenceName = "IDEAL" if not self.referenceAlignment == "IDEAL": referenceName = self.referenceAlignment.name allCompares = config.getCompares() self.__compares = {} if valName in allCompares: self.__compares[valName] = allCompares[valName] else: msg = ("Could not find compare section '%s' in '%s'" % (valName, allCompares)) raise AllInOneError(msg) self.copyImages = copyImages
def replaceByMap(target, the_map): """This function replaces `.oO[key]Oo.` by `the_map[key]` in target. Arguments: - `target`: String which contains symbolic tags of the form `.oO[key]Oo.` - `the_map`: Dictionary which has to contain the `key`s in `target` as keys """ result = target for key in the_map: lifeSaver = 10e3 iteration = 0 while ".oO[" in result and "]Oo." in result: for key in the_map: try: result = result.replace(".oO[" + key + "]Oo.", the_map[key]) except TypeError: #try a dict try: for keykey, value in six.iteritems(the_map[key]): result = result.replace( ".oO[" + key + "['" + keykey + "']]Oo.", value) result = result.replace( ".oO[" + key + '["' + keykey + '"]]Oo.', value) except AttributeError: #try a list try: for index, value in enumerate(the_map[key]): result = result.replace( ".oO[" + key + "[" + str(index) + "]]Oo.", value) except TypeError: raise TypeError( "Something is wrong in replaceByMap! Need a string, dict, or list, but the_map(%s)=%s!" % (repr(key), repr(the_map[key]))) iteration += 1 if iteration > lifeSaver: problematicLines = "" for line in result.splitlines(): if ".oO[" in result and "]Oo." in line: problematicLines += "%s\n" % line msg = ("Oh Dear, there seems to be an endless loop in " "replaceByMap!!\n%s\n%s" % (problematicLines, the_map)) raise AllInOneError(msg) return result
def getCompareStrings(self, requestId=None): result = {} repMap = self.alignmentToValidate.getRepMap() for validationId in self.filesToCompare: repMap["file"] = self.filesToCompare[validationId] if repMap["file"].startswith("/castor/"): repMap["file"] = "rfio:%(file)s" % repMap result[ validationId] = "%(file)s=%(name)s|%(color)s|%(style)s" % repMap if requestId == None: return result else: if not "." in requestId: requestId += ".%s" % GenericValidation.defaultReferenceName if not requestId.split(".")[-1] in result: msg = ("could not find %s in reference Objects!" % requestId.split(".")[-1]) raise AllInOneError(msg) return result[requestId.split(".")[-1]]
def __init__(self, name, config, runGeomComp="1"): self.condShorts = { "TrackerAlignmentErrorExtendedRcd": { "zeroAPE": { "connectString": ("frontier://FrontierProd" "/CMS_CONDITIONS"), "tagName": "TrackerIdealGeometryErrorsExtended210_mc", "labelName": "" } }, "TrackerSurfaceDeformationRcd": { "zeroDeformations": { "connectString": ("frontier://FrontierProd" "/CMS_CONDITIONS"), "tagName": "TrackerSurfaceDeformations_zero", "labelName": "" } }, } section = "alignment:%s" % name if not config.has_section(section): raise AllInOneError, ("section %s not found. Please define the " "alignment!" % section) config.checkInput( section, knownSimpleOptions=['globaltag', 'style', 'color', 'title'], knownKeywords=['condition']) self.name = name if config.exists(section, "title"): self.title = config.get(section, "title") else: self.title = self.name if (int(runGeomComp) != 1): self.name += "_run" + runGeomComp self.title += " run " + runGeomComp if "|" in self.title or "," in self.title or '"' in self.title: msg = "The characters '|', '\"', and ',' cannot be used in the alignment title!" raise AllInOneError(msg) self.runGeomComp = runGeomComp self.globaltag = config.get(section, "globaltag") self.conditions = self.__getConditions(config, section) self.color = config.get(section, "color") self.style = config.get(section, "style")
def __fileListSnippet(self, crab=False, parent=False, firstRun=None, lastRun=None, forcerunselection=False): if crab: files = "" else: splitFileList = list( self.__chunks( self.fileList(firstRun=firstRun, lastRun=lastRun, forcerunselection=forcerunselection), 255)) if not splitFileList: raise AllInOneError( "No files found for dataset {}. Check the spelling, or maybe specify another das instance?" .format(self.__name)) fileStr = ["',\n'".join(files) for files in splitFileList] fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \ for files in fileStr ] files = "\n".join(fileStr) if parent: splitParentFileList = list( self.__chunks( self.fileList(parent=True, firstRun=firstRun, lastRun=lastRun, forcerunselection=forcerunselection), 255)) parentFileStr = [ "',\n'".join(parentFiles) for parentFiles in splitParentFileList ] parentFileStr = [ "secFiles.extend( [\n'" + parentFiles + "'\n] )" \ for parentFiles in parentFileStr ] parentFiles = "\n".join(parentFileStr) files += "\n\n" + parentFiles return files
def __init__(self, valName, alignment, config, configBaseName="TkAlMcValidate", scriptBaseName="TkAlMcValidate", crabCfgBaseName="TkAlMcValidate", resultBaseName="McValidation", outputBaseName="McValidation"): self.configBaseName = configBaseName self.scriptBaseName = scriptBaseName self.crabCfgBaseName = crabCfgBaseName self.resultBaseName = resultBaseName self.outputBaseName = outputBaseName self.needParentFiles = True GenericValidationData.__init__(self, valName, alignment, config, "mcValidate") if self.NJobs > 1: raise AllInOneError( "Parallel jobs not implemented for the MC validation!\n" "Please set parallelJobs = 1.")