ShellStyles.NormalStyle()) # Loop over era, searchMode, and optimizationMode for era in myModuleSelector.getSelectedEras(): for searchMode in myModuleSelector.getSelectedSearchModes(): for optimizationMode in myModuleSelector.getSelectedOptimizationModes( ): #===== Obtain normalization factors myNormFactors = importNormFactors(era, searchMode, optimizationMode, opts.multicrabDir) #===== Nominal module myModuleInfoString = "%s_%s" % (era, searchMode) if len(optimizationMode) > 0: myModuleInfoString += "_%s" % optimizationMode n += 1 print ShellStyles.CaptionStyle( ) + "Module %d/%d: %s/%s%s" % ( n, myTotalModules, myModuleInfoString, shapeType, ShellStyles.NormalStyle()) myStartTime = time.time() nominalModule = ModuleBuilder(opts, myOutputCreator) nominalModule.createDsetMgr( multicrabDir=myMulticrabDir, era=era, searchMode=searchMode, optimizationMode=optimizationMode) if (n == 1): nominalModule.debug() nominalModule.buildModule( _generalOptions["dataSource"], _generalOptions["EWKsource"], myNormFactors["nominal"],
import string from optparse import OptionParser import HiggsAnalysis.NtupleAnalysis.tools.aux as aux import HiggsAnalysis.NtupleAnalysis.tools.ShellStyles as ShellStyles #================================================================================================ # Variable definition #================================================================================================ ss = ShellStyles.SuccessStyle() ns = ShellStyles.NormalStyle() ts = ShellStyles.NoteStyle() hs = ShellStyles.HighlightAltStyle() ls = ShellStyles.HighlightStyle() es = ShellStyles.ErrorStyle() cs = ShellStyles.CaptionStyle() #================================================================================================ # Function definition #================================================================================================ def Verbose(msg, printHeader=False): ''' Calls Print() only if verbose options is set to true. ''' if not opts.verbose: return Print(msg, printHeader) return
def main(): # Object for selecting data eras, search modes, and optimization modes myModuleSelector = analysisModuleSelector.AnalysisModuleSelector() # Obtain multicrab directory myMulticrabDir = "." if opts.mcrab != None: myMulticrabDir = opts.mcrab if not os.path.exists("%s/multicrab.cfg" % myMulticrabDir): msg = "No multicrab directory found at path '%s'! Please check path or specify it with --mcrab!" % ( myMulticrabDir) raise Exception(ShellStyles.ErrorLabel() + msg + ShellStyles.NormalStyle()) if len(opts.shape) == 0: raise Exception( ShellStyles.ErrorLabel() + "Provide a shape identifierwith --shape (for example MT)!" + ShellStyles.NormalStyle()) # Obtain dsetMgrCreator and register it to module selector dsetMgrCreator = dataset.readFromMulticrabCfg(directory=myMulticrabDir) # Obtain systematics names mySystematicsNamesRaw = dsetMgrCreator.getSystematicVariationSources() mySystematicsNames = [] for item in mySystematicsNamesRaw: mySystematicsNames.append("%sPlus" % item) mySystematicsNames.append("%sMinus" % item) if opts.test: mySystematicsNames = [] #[mySystematicsNames[0]] #FIXME # Set the primary source myModuleSelector.setPrimarySource(label=opts.analysisName, dsetMgrCreator=dsetMgrCreator) # Select modules myModuleSelector.doSelect(opts=None) #FIXME: (opts=opts) # Loop over era/searchMode/optimizationMode combos myDisplayStatus = True myTotalModules = myModuleSelector.getSelectedCombinationCount() * ( len(mySystematicsNames) + 1) * len(opts.shape) Verbose("Found %s modules in total" % (myTotalModules), True) count, nEras, nSearchModes, nOptModes, nSysVars = myModuleSelector.getSelectedCombinationCountIndividually( ) if nSysVars > 0: msg = "Will run over %d modules (%d eras x %d searchModes x %d optimizationModes x %d systematic variations)" % ( count, nEras, nSearchModes, nOptModes, nSysVars) else: msg = "Will run over %d modules (%d eras x %d searchModes x %d optimizationModes)" % ( count, nEras, nSearchModes, nOptModes) Print(msg, True) # Create pseudo-multicrab creator myOutputCreator = pseudoMultiCrabCreator.PseudoMultiCrabCreator( opts.analysisName, myMulticrabDir) # Make time stamp for start time myGlobalStartTime = time.time() iModule = 0 # For-loop: All Shapes for shapeType in opts.shape: # Initialize myOutputCreator.initialize(shapeType, prefix="") msg = "Creating dataset for shape \"%s\"%s" % ( shapeType, ShellStyles.NormalStyle()) Verbose(ShellStyles.HighlightStyle() + msg, True) # Get lists of settings erasList = myModuleSelector.getSelectedEras() modesList = myModuleSelector.getSelectedSearchModes() optList = myModuleSelector.getSelectedOptimizationModes() optList.append("") #append the default opt mode! # For-Loop over era, searchMode, and optimizationMode options for era in erasList: for searchMode in modesList: for optimizationMode in optList: Verbose( "era = %s, searchMode = %s, optMode = %s" % (era, searchMode, optimizationMode), True) # If an optimization mode is defined in options skip the rest if opts.optMode != None: if optimizationMode != opts.optMode: continue # Obtain normalization factors myNormFactors = importNormFactors(era, searchMode, optimizationMode, opts.mcrab) # Nominal module myModuleInfoString = getModuleInfoString( era, searchMode, optimizationMode) iModule += 1 # Inform user of what is being processes msg = "Module %d/%d:%s %s/%s" % ( iModule, myTotalModules, ShellStyles.NormalStyle(), myModuleInfoString, shapeType) Print(ShellStyles.CaptionStyle() + msg, True) # Keep time myStartTime = time.time() Verbose("Create dataset manager with given settings", True) nominalModule = ModuleBuilder(opts, myOutputCreator) nominalModule.createDsetMgr(myMulticrabDir, era, searchMode, optimizationMode) if (iModule == 1): if opts.verbose: nominalModule.debug() doQCDNormalizationSyst = False #FIXME if not doQCDNormalizationSyst: msg = "Disabling systematics" Print(ShellStyles.WarningLabel() + msg, True) nominalModule.buildModule(opts.dataSrc, opts.ewkSrc, myNormFactors["nominal"], doQCDNormalizationSyst, opts.normDataSrc, opts.normEwkSrc) if len(mySystematicsNames) > 0: Print( "Adding QCD normalization systematics (iff also other systematics present) ", True) nominalModule.buildQCDNormalizationSystModule( opts.dataSrc, opts.ewkSrc) # FIXME: add quark gluon weighting systematics! if 0: Print("Adding Quark/Gluon weighting systematics", True) nominalModule.buildQCDQuarkGluonWeightingSystModule( opts.dataSrc, opts.ewkSrc, myNormFactors["FakeWeightingUp"], myNormFactors["FakeWeightingDown"], False, opts.normDataSrc, opts.normEwkSrc) Verbose("Deleting nominal module", True) nominalModule.delete() Verbose("Printing time estimate", True) printTimeEstimate(myGlobalStartTime, myStartTime, iModule, myTotalModules) Verbose("Now do the rest of systematics variations", True) for syst in mySystematicsNames: iModule += 1 msg = "Analyzing systematics variations %d/%d: %s/%s/%s" % ( iModule, myTotalModules, myModuleInfoString, syst, shapeType) Print( ShellStyles.CaptionStyle() + msg + ShellStyles.NormalStyle(), True) myStartTime = time.time() systModule = ModuleBuilder(opts, myOutputCreator) # Create dataset manager with given settings systModule.createDsetMgr(myMulticrabDir, era, searchMode, optimizationMode, systematicVariation=syst) # Build asystematics module systModule.buildModule(opts.dataSrc, opts.ewkSrc, myNormFactors["nominal"], False, opts.normDataSrc, opts.normEwkSrc) printTimeEstimate(myGlobalStartTime, myStartTime, iModule, myTotalModules) systModule.delete() Verbose("Pseudo-multicrab ready for %s" % shapeType, True) # Create rest of pseudo multicrab directory myOutputCreator.silentFinalize() # Print some timing statistics Print( "Average processing time per module was %.1f s" % getAvgProcessTimeForOneModule(myGlobalStartTime, myTotalModules), True) Print( "Total elapsed time was %.1f s" % getTotalElapsedTime(myGlobalStartTime), False) msg = "Created pseudo-multicrab %s for shape type \"%s\"" % ( myOutputCreator.getDirName(), shapeType) Print(ShellStyles.SuccessLabel() + msg, True) return
def main(opts): # Object for selecting data eras, search modes, and optimization modes myModuleSelector = analysisModuleSelector.AnalysisModuleSelector() # Obtain dsetMgrCreator and register it to module selector dsetMgrCreator = dataset.readFromMulticrabCfg(directory=opts.mcrab) # Obtain systematics names mySystematicsNamesRaw = dsetMgrCreator.getSystematicVariationSources() mySystematicsNames = [] for i, item in enumerate(mySystematicsNamesRaw, 0): Print( "Using systematic %s" % (ShellStyles.NoteStyle() + item + ShellStyles.NormalStyle()), i == 0) mySystematicsNames.append("%sPlus" % item) mySystematicsNames.append("%sMinus" % item) if opts.test: mySystematicsNames = [] # Set the primary source Verbose( "Setting the primary source (label=%s)" % (ShellStyles.NoteStyle() + opts.analysisName + ShellStyles.NormalStyle()), True) myModuleSelector.setPrimarySource( label=opts.analysisName, dsetMgrCreator=dsetMgrCreator) #fixme: what is label for? # Select modules myModuleSelector.doSelect(opts=None) #fixme: (opts=opts) # Loop over era/searchMode/optimizationMode combos myTotalModules = myModuleSelector.getSelectedCombinationCount() * ( len(mySystematicsNames) + 1) * len(opts.shape) Verbose("Found %s modules in total" % (myTotalModules), True) count, nEras, nSearchModes, nOptModes, nSysVars = myModuleSelector.getSelectedCombinationCountIndividually( ) if nSysVars > 0: msg = "Running over %d modules (%d eras x %d searchModes x %d optimizationModes x %d systematic variations)" % ( count, nEras, nSearchModes, nOptModes, nSysVars) else: msg = "Running over %d modules (%d eras x %d searchModes x %d optimizationModes)" % ( count, nEras, nSearchModes, nOptModes) Verbose(msg, True) # Create pseudo-multicrab creator msg = "Will create pseudo-dataset %s inside the pseudo-multicrab directory" % ( ShellStyles.NoteStyle() + opts.analysisName + ShellStyles.NormalStyle()) Verbose(msg, True) myOutputCreator = pseudoMultiCrabCreator.PseudoMultiCrabCreator( opts.analysisName, opts.mcrab, verbose=opts.verbose) # Make time stamp for start time myGlobalStartTime = time.time() iModule = 0 # For-loop: All Shapes for iShape, shapeType in enumerate(opts.shape, 1): msg = "Shape %d/%d:%s %s" % (iShape, len( opts.shape), ShellStyles.NormalStyle(), shapeType) Print(ShellStyles.CaptionStyle() + msg, True) # Initialize myOutputCreator.initialize( subTitle=shapeType, prefix="") #fixeme: remove shapeType from sub-directory name? # Get lists of settings erasList = myModuleSelector.getSelectedEras() modesList = myModuleSelector.getSelectedSearchModes() optList = myModuleSelector.getSelectedOptimizationModes() if 0: optList.append( "" ) #append the default opt mode iff more optimization modes exist # For-Loop over era, searchMode, and optimizationMode options for era in erasList: for searchMode in modesList: for optimizationMode in optList: Verbose( "era = %s, searchMode = %s, optMode = %s" % (era, searchMode, optimizationMode), True) # If an optimization mode is defined in options skip the rest if opts.optMode != None: if optimizationMode != opts.optMode: continue # Obtain normalization factors myNormFactors = importNormFactors(era, searchMode, optimizationMode, opts.mcrab) # Nominal module myModuleInfoString = getModuleInfoString( era, searchMode, optimizationMode) iModule += 1 # Inform user of what is being processes msg = "Module %d/%d:%s %s/%s" % ( iModule, myTotalModules, ShellStyles.NormalStyle(), myModuleInfoString, shapeType) Print(ShellStyles.CaptionStyle() + msg, True) # Keep time myStartTime = time.time() Verbose("Create dataset manager with given settings", True) nominalModule = ModuleBuilder(opts, myOutputCreator, opts.verbose) nominalModule.createDsetMgr(opts.mcrab, era, searchMode, optimizationMode) if (iModule == 1): if opts.verbose: nominalModule.debug() doQCDNormalizationSyst = False #FIXME if not doQCDNormalizationSyst: msg = "Disabling systematics" Verbose(ShellStyles.WarningLabel() + msg, True) #fixme # Build the module nominalModule.buildModule( opts.dataSrc, opts.ewkSrc, myNormFactors[opts.normFactorKey], doQCDNormalizationSyst, opts.normDataSrc, opts.normEwkSrc) if len(mySystematicsNames) > 0: Print( "Adding QCD normalization systematics (iff also other systematics present) ", True) nominalModule.buildQCDNormalizationSystModule( opts.dataSrc, opts.ewkSrc) # FIXME: add quark gluon weighting systematics! if 0: Print("Adding Quark/Gluon weighting systematics", True) nominalModule.buildQCDQuarkGluonWeightingSystModule( opts.dataSrc, opts.ewkSrc, myNormFactors["FakeWeightingUp"], myNormFactors["FakeWeightingDown"], False, opts.normDataSrc, opts.normEwkSrc) Verbose("Deleting nominal module", True) nominalModule.delete() Verbose("Printing time estimate", True) printTimeEstimate(myGlobalStartTime, myStartTime, iModule, myTotalModules) Verbose("Now do the rest of systematics variations", True) for syst in mySystematicsNames: iModule += 1 msg = "Analyzing systematics variations %d/%d: %s/%s/%s" % ( iModule, myTotalModules, myModuleInfoString, syst, shapeType) Print( ShellStyles.CaptionStyle() + msg + ShellStyles.NormalStyle(), True) myStartTime = time.time() systModule = ModuleBuilder(opts, myOutputCreator) # Create dataset manager with given settings systModule.createDsetMgr(opts.mcrab, era, searchMode, optimizationMode, systematicVariation=syst) # Build asystematics module systModule.buildModule( opts.dataSrc, opts.ewkSrc, myNormFactors[opts.normFactorKey], False, opts.normDataSrc, opts.normEwkSrc) printTimeEstimate(myGlobalStartTime, myStartTime, iModule, myTotalModules) systModule.delete() Verbose("Pseudo-multicrab ready for %s" % shapeType, True) # Print some timing statistics Print( "Average processing time per module was %.1f seconds" % getAvgProcessTimeForOneModule(myGlobalStartTime, myTotalModules), True) Print( "Total elapsed time was %.1f seconds" % getTotalElapsedTime(myGlobalStartTime), False) # Create rest of pseudo multicrab directory myOutputCreator.finalize(silent=False) return