def xsec( self, modified_couplings=None, overwrite=False, skip=False ): key = self.getKey( modified_couplings ) # Do we have the x-sec? if self.xsecDB.contains(key) and not overwrite: logger.debug( "Found x-sec %s for key %r. Do nothing.", self.xsecDB.get(key), key ) return self.xsecDB.get(key) elif skip: return u_float(0) else: print "Trying to get xsec" self.__initialize( modified_couplings ) logger.info( "Calculating x-sec" ) # rerun MG to obtain the correct x-sec (with more events) with open( os.path.join( self.processTmpDir, 'Cards/run_card.dat' ), 'a' ) as f: f.write( ".false. = gridpack\n" ) logger.info( "Calculate x-sec: Calling bin/generate_events" ) output = subprocess.check_output( [ os.path.join( self.processTmpDir, 'bin/generate_events' ) , '-f' ] ) for i in range(10): try: output = subprocess.check_output( [ os.path.join( self.processTmpDir, 'bin/generate_events' ) , '-f' ] ) m = re.search( "Cross-section :\s*(.*) \pb", output ) logger.info( "x-sec: {} pb".format(m.group(1)) ) break except ValueError: logger.info("Encountered problem during the MG run. Restarting.") xsec_ = u_float.fromString(m.group(1)) self.xsecDB.add( key, xsec_, overwrite=True ) logger.info( "Done!" ) return xsec_
def observation(self, region, channel, setup, overwrite): if setup.nJet == "3p": setup4p = setup.sysClone(parameters={"nJet": (4, -1)}) setup3 = setup.sysClone(parameters={"nJet": (3, 3)}) return sum([ self.cachedEstimate(region, channel, s, overwrite=overwrite) for s in [setup3, setup4p] ]) if channel == "all": return sum([ self.cachedEstimate(region, c, setup, overwrite=overwrite) for c in lepChannels ]) elif channel == "SFtight": return sum([ self.cachedEstimate(region, c, setup, overwrite=overwrite) for c in dilepChannels ]) else: preSelection = setup.preselection("Data", channel=channel) # cut = "&&".join([region.cutString(setup.sys['selectionModifier']), preSelection['cut']]) cut = "&&".join([region.cutString(), preSelection['cut']]) logger.debug("Using cut %s" % cut) weight = preSelection['weightStr'] if hasattr(setup, "blinding") and setup.blinding: weight += "*" + setup.blinding return u_float(**self.process.getYieldFromDraw( selectionString=cut, weightString=weight))
def cachedTemplate(self, selection, weight='(1)', save=True, overwrite=False): key = { "selection": selection, "weight": weight, "source": self.source_sample.name } if (self.cache and self.cache.contains(key)) and not overwrite: result = self.cache.get(key) logger.info("Loaded MC PU profile from %s" % (self.cache.database_file)) logger.debug("Key used: %s result: %r" % (key, result)) elif self.cache: logger.info("Obtain PU profile for %s" % (key, )) result = self.makeTemplate(selection=selection, weight=weight) if result: result = self.cache.addData(key, result, overwrite=save) logger.info("Adding PU profile to cache for %s : %r" % (key, result)) else: logger.warning( "Couldn't create PU profile to cache for %s : %r" % (key, result)) else: result = self.makeTemplate(selection=selection, weight=weight) return result
def wrapper(arg, y): r, channel, est, estimate = arg setup = setups[y][est] logger.debug( "Running transfer factor for region %s, channel %s in setup %s for QCD-DD" % (r, channel, est)) estimate.initCache(setup.defaultCacheDir()) res = estimate.cachedTransferFactor(r, channel, setup, checkOnly=True) return (arg, res)
def preselection(self, dataMC, channel="all", processCut=None): """Get preselection cutstring.""" cut = self.selection(dataMC, channel=channel, processCut=processCut, **self.parameters) logger.debug("Using cut-string: %s", cut) if processCut: logger.info("Adding process specific cut: %s" % processCut) return cut
def writeToCache(self, region, channel, setup, value, signalAddon=None, save=True, overwrite=False, checkOnly=False): key = self.uniqueKey(region, channel, setup, signalAddon=signalAddon) if (self.cache and self.cache.contains(key)) and not overwrite: res = self.cache.get(key) # if res.val != value.val: print "Warning, caches estimate not equal to input value: have %s, got %s"%(res, value) # logger.debug( "Loading cached %s result for %r : %r"%(self.name, key, res) ) elif self.cache and not checkOnly: _res = self.cache.add( key, value, overwrite=True ) res = value logger.debug( "Adding cached %s result for %r : %r" %(self.name, key, res) ) else: res = u_float(-1,0) return res if res >= 0 or checkOnly else u_float(0,0)
def __initialize( self, modified_couplings = None ): # Initialize setup self.config.initialize( modified_couplings ) # Write process card self.__writeProcessCard() logger.info( "Running MG executable: python %s -f %s", self.config.MG5_tmpdir + '/bin/mg5_aMC', self.tmpProcessCard ) subprocess.check_output( [ "python", self.config.MG5_tmpdir + '/bin/mg5_aMC', '-f', self.tmpProcessCard ] ) logger.info( "Done with MG executable" ) # copy files from central gridpack for filename in [ 'grid_card.dat', 'me5_configuration.txt', ]: logger.info( "Copying files from GP directory to temporary process directory: %s", filename ) source = os.path.join( self.config.GP_tmpdir, 'process/madevent/Cards', filename ) target = os.path.join( self.processTmpDir, 'Cards', filename ) shutil.copyfile( source, target ) logger.debug( "Done with %s -> %s", source, target ) if self.run_card: logger.info( "Copying provided run card: %s", self.run_card ) target = os.path.join( self.processTmpDir, 'Cards', "run_card.dat" ) shutil.copyfile( self.run_card, target ) logger.debug( "Done with %s -> %s", source, target ) else: filename = "run_card.dat" logger.info( "Copying files from GP directory to temporary process directory: %s", filename ) source = os.path.join( self.config.GP_tmpdir, 'process/madevent/Cards', filename ) target = os.path.join( self.processTmpDir, 'Cards', filename ) shutil.copyfile( source, target ) logger.debug( "Done with %s -> %s", source, target ) # copy reweight cards if self.reweight: source = os.path.join( self.config.data_path, 'template', 'template_reweight_card_' + self.config.model_name + '.dat' ) target = os.path.join( self.processTmpDir, 'Cards', 'reweight_card.dat' ) print target shutil.copyfile( source, target ) if os.path.isfile( target ): logger.debug( "Done with %s -> %s", source, target ) else: logger.info( "File copy failed. WTF!" ) # Append to me5_configuration.txt with open( os.path.join( self.processTmpDir, 'Cards/me5_configuration.txt' ), 'a') as f: f.write( "run_mode = 2\n" ) f.write( "nb_core = 4\n" ) f.write( "lhapdf = /cvmfs/cms.cern.ch/%s/external/lhapdf/6.1.6/share/LHAPDF/../../bin/lhapdf-config\n" % os.environ["SCRAM_ARCH"] ) f.write( "automatic_html_opening = False\n" ) # Append to run_card.dat with open( os.path.join( self.processTmpDir, 'Cards/run_card.dat' ), 'a' ) as f: f.write( "{} = nevents\n".format(self.nEvents) )
def initialize(self, modified_couplings=None): ''' Update the restriction card ''' logger.info("#################### Model Setup ######################") self.__pre_initialize() # couplings modified_couplings = modified_couplings if modified_couplings is not None else {} # Check whether couplings are in the model for coup in modified_couplings.keys(): if coup not in self.all_model_couplings: logger.error( "Coupling %s not found in model %s. All available couplings: %s", coup, self.model_name, ",".join(self.all_model_couplings)) raise RuntimeError logger.debug('Creating restriction file based on template %s', self.restrictCardTemplate) # make block strings to be inserted into template file block_strings = {} for block in self.model.keys(): # copy defaults couplings = copy.deepcopy(self.model[block]) # make modifications & build string for the template file block_strings[block + '_template_string'] = "" for i_coupling, coupling in enumerate( couplings): # coupling is a pair (name, value) if modified_couplings.has_key(coupling[0]): coupling[1] = modified_couplings[coupling[0]] block_strings[block + '_template_string'] += "%6i %8.6f # %s\n" % ( i_coupling + 1, coupling[1], coupling[0]) # read template file with open(self.restrictCardTemplate, 'r') as f: template_string = f.read() out = open(self.restrictCard, 'w') out.write(template_string.format(**block_strings)) out.close() logger.info('Written restriction file %s', self.restrictCard) logger.info("################# Done: Model Setup ###################")
def wrapper(arg): # r,channel,set = arg channel, set = arg logger.debug("Running transfer factor, channel %s in setup %s for QCD-DD" % (channel, args.controlRegion)) res = estimate.cachedTransferFactor(channel, setup, qcdUpdates=args.customQCDUpdates, save=True, overwrite=args.overwrite, checkOnly=args.checkOnly) _ = estimate.cachedQCDMCTransferFactor(channel, setup, qcdUpdates=args.customQCDUpdates, save=True, overwrite=args.overwrite, checkOnly=args.checkOnly) return (arg, res)
def __pre_initialize(self): ''' Create temporary directories and unzip GP. Time consuming. ''' if self.__isPreInitialized: logger.debug("Already pre-initialized in %s. Do nothing.", self.uniquePath) return # Now begin with the work logger.info("############# pre-initialize Configuration ############") os.makedirs(self.uniquePath) # create new directories os.makedirs(self.GP_tmpdir) os.makedirs(self.MG5_tmpdir) # unzip MG tarball logger.info("Extracting madgraph") subprocess.call( ['tar', 'xaf', self.MG5_tarball, '--directory', self.uniquePath]) # unzip gridpack for central config files logger.info("Extracting central gridpack") subprocess.call( ['tar', 'xaf', self.GP_tarball, '--directory', self.GP_tmpdir]) # copy private UFO files from models in repository if not os.path.isdir(self.MG5_tmpdir + "/Models/" + self.model_name): logger.info("Copying UFO from private Model database for model %s", self.model_name) src_dir = os.path.expandvars( '$CMSSW_BASE/src/TTGammaEFT/Generation/data/models/%s/UFO' % self.model_name) target_dir = os.path.join(self.MG5_tmpdir, "models", self.model_name) logger.debug("Copying UFO from %s to %s", src_dir, target_dir) shutil.copytree(src_dir, target_dir) else: logger.info("Using UFO from MG5 for model %s", self.model_name) self.__isPreInitialized = True logger.info("########## Done: pre-initialize Configuration #########")
def cachedObservation(self, region, channel, setup, save=True, overwrite=False, checkOnly=False): key = self.uniqueKey(region, channel, setup) if (self.cache and self.cache.contains(key)) and not overwrite: res = self.cache.get(key) logger.debug("Loading cached %s result for %r : %r" % (self.name, key, res)) elif self.cache and not checkOnly: res = self.observation(region, channel, setup, overwrite) _res = self.cache.add(key, res, overwrite=True) logger.debug("Adding cached %s result for %r" % (self.name, key)) elif not checkOnly: res = self.observation(region, channel, setup, overwrite) else: res = u_float(-1, 0) return res if res >= 0 or checkOnly else u_float(0, 0)
def wrapper(arg): r, c, s, inclusive = arg print r, c, s.sys["reweight"], inclusive logger.debug("Calculating estimate for %s in region %s and channel %s" % (args.selectEstimator, r, c)) if inclusive: res = estimateIncl.cachedEstimate(r, c, s, save=True, overwrite=args.overwrite, checkOnly=args.checkOnly) return (estimateIncl.uniqueKey(r, c, s), res) else: res = estimate.cachedEstimate(r, c, s, save=True, overwrite=args.overwrite, checkOnly=args.checkOnly) return (estimate.uniqueKey(r, c, s), res)
def weightString(self, dataMC, photon="PhotonGood0", addMisIDSF=False): lumiString = "(35.92*(year==2016)+41.53*(year==2017)+59.74*(year==2018))" _weightString = {} _weightString["Data"] = "weight" _weightString["MC"] = "*".join([self.sys["weight"]] + ( self.sys["reweight"] if self.sys["reweight"] else [])) _weightString["MC"] += "*%s" % lumiString if addMisIDSF and photon: if self.nJet == "2p": misIDSF_val = misID2pSF_val elif self.nJet == "3p": misIDSF_val = misID3pSF_val elif self.nJet == "4p": misIDSF_val = misID4pSF_val elif self.nJet == "2": misIDSF_val = misID2SF_val elif self.nJet == "3": misIDSF_val = misID3SF_val elif self.nJet == "4": misIDSF_val = misID4SF_val elif self.nJet == "5": misIDSF_val = misID5SF_val if self.year == "RunII": ws = "(%s)" % _weightString["MC"] ws16 = "+(%s*(%s_photonCatMagic==2)*(%f-1)*(year==2016))" % ( _weightString["MC"], photon, misIDSF_val[2016].val) ws17 = "+(%s*(%s_photonCatMagic==2)*(%f-1)*(year==2017))" % ( _weightString["MC"], photon, misIDSF_val[2017].val) ws18 = "+(%s*(%s_photonCatMagic==2)*(%f-1)*(year==2018))" % ( _weightString["MC"], photon, misIDSF_val[2018].val) _weightString["MC"] = ws + ws16 + ws17 + ws18 else: _weightString["MC"] += "+%s*(%s_photonCatMagic==2)*(%f-1)" % ( _weightString["MC"], photon, misIDSF_val[self.year].val) if dataMC == "DataMC": return _weightString if dataMC == "Data": _weightString = _weightString["Data"] elif (dataMC == "MC" or dataMC == "MCpTincl"): _weightString = _weightString["MC"] logger.debug("Using weight-string: %s", _weightString) return _weightString
def cachedFakeFactor(self, region, channel, setup, overwrite=False, checkOnly=False): key = self.uniqueKey(region, channel, setup) if (self.helperCache and self.helperCache.contains(key)) and not overwrite: res = self.helperCache.get(key) logger.debug( "Loading cached %s result for %r : %r"%(self.name, key, res) ) elif self.helperCache and not checkOnly: logger.debug( "Calculating %s result for %r"%(self.name, key) ) res = self._dataDrivenFakeCorrectionFactor( region, channel, setup, overwrite=overwrite ) _res = self.helperCache.add( key, res, overwrite=True ) logger.debug( "Adding cached transfer factor for %r : %r" %(key, res) ) elif not checkOnly: res = self._dataDrivenFakeCorrectionFactor( region, channel, setup, overwrite=overwrite ) else: res = u_float(-1,0) return res if res > 0 or checkOnly else u_float(0,0)
def cachedEstimate(self, region, channel, setup, signalAddon=None, save=True, overwrite=False, checkOnly=False): key = self.uniqueKey(region, channel, setup, signalAddon=signalAddon) if (self.cache and self.cache.contains(key)) and not overwrite: res = self.cache.get(key) logger.debug( "Loading cached %s result for %r : %r"%(self.name, key, res) ) elif self.cache and not checkOnly: logger.debug( "Calculating %s result for %r"%(self.name, key) ) res = self._estimate( region, channel, setup, signalAddon=signalAddon, overwrite=overwrite ) _res = self.cache.add( key, res, overwrite=True ) logger.debug( "Adding cached %s result for %r : %r" %(self.name, key, res) ) elif not checkOnly: res = self._estimate( region, channel, setup, signalAddon=signalAddon, overwrite=overwrite) else: res = u_float(-1,0) return res if res >= 0 or checkOnly else u_float(0,0)
def cachedQCDMCTransferFactor(self, channel, setup, qcdUpdates=None, save=True, overwrite=False, checkOnly=False): key = self.uniqueKey("regionQCDMC", channel, setup, qcdUpdates=qcdUpdates) if (self.tfCache and self.tfCache.contains(key)) and not overwrite: res = self.tfCache.get(key) logger.debug( "Loading cached %s result for %r : %r"%(self.name, key, res) ) elif self.tfCache and not checkOnly: logger.debug( "Calculating %s result for %r"%(self.name, key) ) # res = self._dataDrivenTransferFactor( channel, setup, qcdUpdates=qcdUpdates, overwrite=overwrite ) res = self._transferFactor( channel, setup, qcdUpdates=qcdUpdates, overwrite=overwrite ) _res = self.tfCache.add( key, res, overwrite=True ) logger.debug( "Adding cached transfer factor for %r : %r" %(key, res) ) elif not checkOnly: # res = self._dataDrivenTransferFactor( channel, setup, qcdUpdates=qcdUpdates, overwrite=overwrite ) res = self._transferFactor( channel, setup, qcdUpdates=qcdUpdates, overwrite=overwrite ) else: res = u_float(-1,0) return res if res > 0 or checkOnly else u_float(0,0)
default=0, help="Run only job i") argParser.add_argument('--sample', action='store', default='TTG_4WC_ref', type=str, help="Which sample to plot") args = argParser.parse_args() # Logger import Analysis.Tools.logger as logger import RootTools.core.logger as logger_rt logger = logger.get_logger(args.logLevel, logFile=None) logger_rt = logger_rt.get_logger(args.logLevel, logFile=None) logger.debug("Start run_estimate.py") if args.checkOnly: args.overwrite = False if not args.controlRegion: logger.warning("ControlRegion not known") sys.exit(0) # load and define the EFT sample from TTGammaEFT.Samples.genTuples_TTGamma_EFT_postProcessed import * eftSample = eval(args.sample) #settings for eft reweighting w = WeightInfo(eftSample.reweight_pkl) w.set_order(args.order) variables = w.variables
def _estimate(self, region, channel, setup, signalAddon=None, overwrite=False): ''' Concrete implementation of abstract method 'estimate' as defined in Systematic ''' logger.debug("MC prediction for %s channel %s" % (self.name, channel)) if setup.nJet == "3p": setup4p = setup.sysClone(parameters={"nJet": (4, -1)}) setup3 = setup.sysClone(parameters={"nJet": (3, 3)}) return sum([ self.cachedEstimate(region, channel, s, signalAddon=signalAddon, overwrite=overwrite) for s in [setup3, setup4p] ]) if channel == 'all': # 'all' is the total of all contributions return sum([ self.cachedEstimate(region, c, setup, signalAddon=signalAddon, overwrite=overwrite) for c in lepChannels ]) elif channel == 'SFtight': # 'SFtight' is the total of mumutight and eetight contributions return sum([ self.cachedEstimate(region, c, setup, signalAddon=signalAddon, overwrite=overwrite) for c in dilepChannels ]) else: # change the sample processed if there is a signal addon like TuneUp if signalAddon: if self.name.split("_")[-1] in [ "gen", "misID", "had", "hp", "fake", "PU" ]: name = "_".join( self.name.split("_")[:-1] + [signalAddon, self.name.split("_")[-1]]) else: name = "_".join([self.name, signalAddon]) setattr(self, "process" + signalAddon, setup.processes[name]) # preSelection = setup.preselection('MC' if not signalAddon else "MCpTincl", channel=channel, processCut=self.processCut) preSelection = setup.preselection('MC', channel=channel, processCut=self.processCut) cuts = [ region.cutString(setup.sys['selectionModifier']), preSelection['cut'] ] # if setup.parameters["photonIso"] and setup.parameters["photonIso"] != "lowChgIsolowSieie": # self.processCut = self.processCut.replace("photoncat", "photonhadcat") # if self.processCut: # cuts.append( cutInterpreter.cutString(self.processCut) ) # logger.info( "Adding process specific cut %s"%self.processCut ) cut = "&&".join(cuts) weight = preSelection['weightStr'] logger.debug("Using cut %s and weight %s" % (cut, weight)) # return setup.lumi/1000.*u_float(**getattr(self,"".join(["process",signalAddon if signalAddon else ""])).getYieldFromDraw(selectionString = cut, weightString = weight) ) # print cut, weight return u_float(**getattr( self, "".join(["process", signalAddon if signalAddon else ""]) ).getYieldFromDraw(selectionString=cut, weightString=weight))
try: os.makedirs( targetPath ) except: pass nExistingFiles = len(os.listdir(targetPath)) if nExistingFiles > options.nJobs: raise Exception("Error: More files exist in target directory as should be processed! Check your nJobs input! Got nJobs %i, existing files: %i"%(options.nJobs, nExistingFiles)) logger.info("%i files exist in target directory! Processing %i files."%(nExistingFiles, options.nJobs)) # Single file post processing if options.fileBasedSplitting or options.nJobs > 1: len_orig = len(sample.files) sample = sample.split( n=options.nJobs, nSub=options.job) if sample is None: logger.info( "No such sample. nJobs %i, job %i numer of files %i", options.nJobs, options.job, len_orig ) sys.exit(0) logger.info( "fileBasedSplitting: Run over %i/%i files for job %i/%i."%(len(sample.files), len_orig, options.job, options.nJobs)) logger.debug( "fileBasedSplitting: Files to be run over:\n%s", "\n".join(sample.files) ) # Directories outputFilePath = os.path.join( output_directory, sample.name + '.root' ) targetFilePath = os.path.join( targetPath, sample.name + '.root' ) filename, ext = os.path.splitext( outputFilePath ) if os.path.exists( output_directory ) and options.overwrite: if options.nJobs > 1: logger.warning( "NOT removing directory %s because nJobs = %i", output_directory, options.nJobs ) else: logger.info( "Output directory %s exists. Deleting.", outputFilePath ) shutil.rmtree( outputFilePath, ignore_errors=True ) if not os.path.exists( output_directory ): try:
if os.path.exists( targetFileName ): if options.overwrite: logger.info( "Found file %s. Overwriting.", targetFileName ) cmd.append( '-f' ) else: logger.info( "File %s already exists. Skipping.", targetFileName ) cmd = None from subprocess import call if cmd is not None: call(cmd + [targetFileName] + job) if options.delete: isOK = checkRootFile( targetFileName, checkForObjects = [options.treeName]) \ if options.treeName is not None else checkRootFile( targetFileName ) if isOK: for f in job: os.remove( f ) logger.debug( "Deleted input %s", f) else: logger.warning( "File %s does not look OK. Checked for tree: %r. Did not delete input.", targetFileName, options.treeName ) else: if not isOK: logger.warning( "File %s does not look OK. Checked for tree: %r.", targetFileName, options.treeName ) if options.really: logger.info("Done.") else: logger.info("Done with nothing. Use --really to hadd and --delete to delete the input.")
help="Number of Events") argParser.add_argument('--logLevel', action='store', default='INFO', nargs='?', choices=defaultLogger, help="Log level for logging") argParser.add_argument('--makeGridpack', action='store_true', help="make gridPack?") argParser.add_argument('--calcXSec', action='store_true', help="calculate x-sec?") args = argParser.parse_args() logger.debug("Coupling arguments: %r", args.couplings) if not args.runCard.endswith(".dat"): args.runCard += ".dat" runCardPath = os.path.join(run_path, args.runCard) # Single argument -> interpret as file if len(args.couplings) == 1 and os.path.isfile(args.couplings[0]): with open(args.couplings[0], 'r') as f: param_points = [line.rstrip().split() for line in f.readlines()] # Interpret couplings elif len(args.couplings) != 1: # make a list of the form [ ['c1', v1, v2, ...], ['c2', ...] ] so we can recurse in the couplings c1,c2,... coupling_list = [] for a in args.couplings: try:
}) return dictList # Load File logger.info("Now running on pp file %s and checking in directory %s", args.file, args.data_directory) file = os.path.expandvars("$CMSSW_BASE/src/tWZ/postProcessing/%s.sh" % args.file) dictList = getDataDictList(file) isData = "Run" in args.file execCommand = [] for ppEntry in dictList: sample = ppEntry['sample'] logger.debug("Checking sample %s" % sample) postfix = "" # check whether we do reduction: if "--reduceSizeBy" in ppEntry["command"]: postfix += "_redBy%i" % int(ppEntry["command"].split('--reduceSizeBy') [1].lstrip().split(' ')[0]) dirPath = os.path.join(args.data_directory, ppEntry["dir"], ppEntry["skim"], sample + postfix) # check whether we have an LHE HT cut: if "--LHEHTCut" in ppEntry["command"]: postfix += "_lheHT%i" % int( ppEntry["command"].split('--LHEHTCut')[1].lstrip().split(' ')[0]) dirPath = os.path.join(args.data_directory, ppEntry["dir"], str(ppEntry["year"]), ppEntry["skim"], sample + postfix)
logger = logging.getLogger(__name__) if args.file.endswith(".sh"): args.file = args.file.rstrip(".sh") # Load File logger.info("Now running on pp file %s" % args.file) file = os.path.expandvars("$CMSSW_BASE/src/TTGammaEFT/postprocessing/%s.sh" % args.file) dictList = getDataDictList(file) isData = "Run" in args.file execCommand = [] allFiles = [] for ppEntry in dictList: sample = ppEntry['sample'] logger.debug("Checking sample %s" % sample) # Check whether file exists on DPM, no check if root file is ok implemented for now if "clip" in hostname.lower(): # dirPath = os.path.join( postprocessing_output_directory, ppEntry["dir"], ppEntry["skim"], sample ) dirPath = os.path.join(dpm_directory, "postprocessed", ppEntry["dir"], ppEntry["skim"], sample) files = os.listdir(dirPath) if os.path.exists(dirPath) else [] rootFiles = filter( lambda file: file.endswith(".root") and not file.startswith( "nanoAOD"), files) rootFiles = [item.split(".root")[0] for item in rootFiles] allFiles += [os.path.join(dirPath, r + ".root") for r in rootFiles] else: dirPath = os.path.join(dpm_directory, 'postprocessed', ppEntry["dir"],
def make_batch_job( batch_job_file, batch_job_title, batch_output_dir , command ): # If X509_USER_PROXY is set, use existing proxy. if options.dpm: if host == 'lxplus': from StopsDilepton.Tools.user import cern_proxy_certificate proxy_location = cern_proxy_certificate else: proxy_location = None from RootTools.core.helpers import renew_proxy proxy = renew_proxy( proxy_location ) logger.info( "Using proxy certificate %s", proxy ) proxy_cmd = "export X509_USER_PROXY=%s"%proxy else: proxy_cmd = "" import subprocess if host == 'heplx': template =\ """\ #!/bin/sh -x #SBATCH -J {batch_job_title} #SBATCH -D {pwd} #SBATCH -o {batch_output_dir}batch-test.%j.out {proxy_cmd} voms-proxy-info -all eval \`"scram runtime -sh"\` echo CMSSW_BASE: {cmssw_base} echo Executing user command echo "{command}" {command} voms-proxy-info -all """.format(\ command = command, cmssw_base = os.getenv("CMSSW_BASE"), batch_output_dir = batch_output_dir, batch_job_title = batch_job_title, pwd = os.getenv("PWD"), proxy_cmd = proxy_cmd ) elif host == 'lxplus': template =\ """\ #!/bin/bash export CMSSW_PROJECT_SRC={cmssw_base}/src cd $CMSSW_PROJECT_SRC eval `scramv1 ru -sh` alias python={python_release} which python python --version {proxy_cmd} voms-proxy-info -all echo CMSSW_BASE: $CMSSW_BASE cd {pwd} echo Executing user command while in $PWD echo "{command}" {command} voms-proxy-info -all """.format(\ command = command, cmssw_base = os.getenv("CMSSW_BASE"), #batch_output_dir = batch_output_dir, #batch_job_title = batch_job_title, pwd = os.getenv("PWD"), proxy_cmd = proxy_cmd, python_release = subprocess.check_output(['which', 'python']).rstrip(), ) batch_job = file(batch_job_file, "w") batch_job.write(template) batch_job.close() logger.debug("Local batch job file: %s", batch_job_file) logger.debug("Batch job:\n%s", template) return
checkOnly=args.checkOnly) return (estimateIncl.uniqueKey(r, c, s), res) else: res = estimate.cachedEstimate(r, c, s, save=True, overwrite=args.overwrite, checkOnly=args.checkOnly) return (estimate.uniqueKey(r, c, s), res) jobs = [] for region in regions: logger.debug("Queuing jobs for region %s", region) for c in channels: logger.debug("Queuing jobs for channel %s", c) jobs.append((region, c, setup, False)) for var in variations: jobs.append( (region, c, setup.sysClone(sys={"reweight": [var]}), False)) logger.debug("Queuing jobs for channel %s", c) jobs.append((noRegions[0], "all", setupIncl, True)) for var in variations: jobs.append((noRegions[0], "all", setupIncl.sysClone(sys={"reweight": [var]}), True)) logger.info("Created %s jobs", len(jobs))
def loopOverLogs( directory, filterCommand=None, addon=None, ignore="" ): logger.info("Checking log files in directory %s"%directory ) files = list(filter(os.path.isfile, glob.glob(directory + "/*"))) # files = list(filter(os.path.isfile, glob.glob(directory + "/*.err"))) # sort logs by latest one files.sort(key=lambda x: -os.path.getmtime(x)) # filter logs by latest cmd filteredFiles = [] filteredCmds = [] for file in files: cmd = getCommandFromLog(file.replace(".err",".out")) if not cmd: logger.debug( "Could not find command for file %s"%file ) continue # filter commands if filterCommand and not all( [c in cmd for c in filterCommand.split("*")] ): continue # check if a later version of the cmd has already been executed if cmd.replace(ignore,"").replace(" ","") not in [f.replace(ignore,"").replace(" ","") for f in filteredCmds]: filteredFiles.append(file) filteredCmds.append(cmd) logger.info("Loop over %i file%s"%(len(filteredFiles),"s" if len(filteredFiles)>1 else "") ) # Loop over error filteredFiles for i, file in enumerate(filteredFiles): # filter for scripts like nanopostprocessing.py cmd = filteredCmds[i] with open( file, "r" ) as f: logs = f.readlines() logger.debug("Loop over %i entries in file"%len(logs)) # Loop over error lines for log in logs: if any( word in log.lower() for word in trigger_words ): # Found error! logger.debug("Found error '%s' in file %s"%(log.split("\n")[0], file) ) if any( expression.lower() in log.lower() for expression in accepted_errors ): # Error is ok! logger.debug("Error is accepted!") continue # Found uncaught error! logger.info("Error in file %s:\t\t%s"%(file, log[:80].replace("\n"," "))) # logging error files and commands with open( "errorCommands.sh", "a" ) as ferr: ferr.write( " ".join([cmd, addon if addon else "", "\n"]) ) with open( "errorLogs.log", "a" ) as ferr: ferr.write( file + "\n" ) # Skipping rest of the log file break
argParser.add_argument("--checkOnly", action="store_true", help="check values?") #argParser.add_argument('--nJobs', action='store', default=1, type=int, help="Maximum number of simultaneous jobs.") #argParser.add_argument('--job', action='store', default=0, type=int, help="Run only job i") args = argParser.parse_args() if args.year != "RunII": args.year = int(args.year) # Logging import Analysis.Tools.logger as logger logger = logger.get_logger(args.logLevel, logFile=None) import RootTools.core.logger as logger_rt logger_rt = logger_rt.get_logger(args.logLevel, logFile=None) logger.debug("Start run_transferFactor.py") if args.customQCDUpdates: args.customQCDUpdates = customQCDTF_updates[args.customQCDUpdates] if not args.controlRegion: logger.warning("ControlRegion not known") sys.exit(0) parameters = allRegions[args.controlRegion]["parameters"] channels = [args.mode] if args.mode != "all" else [ "e", "mu" ] #allRegions[args.controlRegion]["channels"] #regions = allRegions[args.controlRegion]["inclRegion"] #if allRegions[args.controlRegion]["noPhotonCR"] else allRegions[args.controlRegion]["inclRegion"] + allRegions[args.controlRegion]["regions"] + regionsTTG20To120 + regionsTTG120To220 + regionsTTG220 setup = Setup(year=args.year, photonSelection=False,