def mk_secondary_lumimask(dset): dq = das_query("file dataset=%s instance=prod/phys03" % dset, cmd='dasgoclient --dasmaps=./') assert 'data' in dq.keys() fs = [str(f['file'][0]['name']) for f in dq['data']] #fs = fs[:2] print('N files:', len(fs)) lumis = [] dqs = [ das_query("lumi file=%s instance=prod/phys03" % f, cmd='dasgoclient --dasmaps=./') for f in fs ] for dq in dqs: for data in dq['data']: for lumi in data['lumi'][0]['lumi_section_num']: lumis.append([data['lumi'][0]['run_number'], lumi]) jsonList = LumiList(lumis=lumis) #print(jsonList) output_file = dset.split('/')[2].split('-')[1].split('_')[0] #print(output_file) jsonList.writeJSON(output_dir + output_file + '_3photons_imgskim_lumi_list.json')
def addToDataset(self, catalog, dsetName, files): if dsetName in catalog: if self.force_: catalog[dsetName]["files"] = files else: self.mergeDataset(catalog[dsetName], {"files": files}) #---Recover missing info if "dset_type" not in catalog[ dsetName] or not catalog[dsetName]["dset_type"]: dset_type = das_query( "datatype dataset=%s instance=prod/phys03" % dsetName) catalog[dsetName][ "dset_type"] = dset_type['data'][0]['datatype'][0][ 'data_type'] if 'data' in dset_type else None if ("parent_n_units" not in catalog[dsetName] or catalog[dsetName]["parent_n_units"] == None) and catalog[dsetName]["dset_type"] != None: catalog[dsetName]["parent_n_units"] = self.getParentInfo( catalog[dsetName]["dset_type"], dsetName) else: #---First import dset_type = das_query("datatype dataset=%s instance=prod/phys03" % dsetName) dset_type = dset_type['data'][0]['datatype'][0][ 'data_type'] if 'data' in dset_type else None parent_info = self.getParentInfo(dset_type, dsetName) if dset_type else None catalog[dsetName] = { "files": files, "parent_n_units": parent_info, "dset_type": dset_type }
def getParentInfo(self, dset_type, dsetName): parent_n_info = 'nlumis' if dset_type=='data' else 'nevents' parent_dset = das_query("parent dataset=%s instance=prod/phys03" % dsetName)['data'][0]['parent'][0]['name'] parent_info = das_query("dataset dataset=%s instance=prod/phys03" % parent_dset) try: parent_info = parent_info['data'][-1]['dataset'][0][parent_n_info] except KeyError: try: parent_info = das_query("dataset dataset=%s" % parent_dset) parent_info = parent_info['data'][-1]['dataset'][0][parent_n_info] except KeyError: parent_info = None return parent_info
def getFilesFomDAS(self,dsetName): """ Read dataset files from DAS. @dsetName: dataset name """ ## response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) ## response = das_query("https://cmsweb.cern.ch","file dataset=%s instance=%s | grep file.name,file.nevents" % (dsetName,self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) response = das_query("file dataset=%s instance=%s | grep file.name,file.nevents" % (dsetName,self.dbs_instance_)) files=[] for d in response["data"]: for jf in d["file"]: if "nevents" in jf: files.append({ "name" : jf["name"], "nevents" : jf["nevents"] }) break return files
def getFilesFomDAS(self,dsetName): """ Read dataset files from DAS. @dsetName: dataset name """ ## response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) ## response = das_query("https://cmsweb.cern.ch","file dataset=%s instance=%s | grep file.name,file.nevents" % (dsetName,self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) response = das_query("file dataset=%s instance=%s | grep file.name,file.nevents" % (dsetName,self.dbs_instance_)) files=[] for d in response["data"]: for jf in d["file"]: if "nevents" in jf: files.append({ "name" : jf["name"], "nevents" : jf["nevents"] }) break return files
def file_from_das(dataset): if options.use_parent: format_ = dataset.split("/")[2] if 'SIM' in format_: dataset.replace("NANOAODSIM", "MINIAODSIM") else: dataset.replace("NANOAOD", "MINIAOD") response = das_query("file dataset=%s | grep file.name,file.nevents" % (dataset)) root_file_list = [] nevents = 0 for d in response.get("data", []): for jf in d["file"]: if "nevents" in jf: nevents += jf["nevents"] root_file_list.append({ "name": str(jf["name"]), "nevents": int(jf["nevents"]) }) break return root_file_list, nevents
def importFromDAS(self, list_datasets): """ Import datasets from DAS to the catalog. @datasets: wildecard to be usd in dataset query """ # ret, out = commands.getstatusoutput("voms-proxy-info -e") if ret != 0: print "\n\nNo valid voms proxy found. This is needed to query DAS.\nPlease create a valid proxy running the following command:\nvoms-proxy-init -voms cms\n" sys.exit(-1) catalog = self.readCatalog() print "Importing from das %s" % list_datasets datasets = [] for dataset in list_datasets: if "*" in dataset: # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) response = das_query( "dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_)) ## print response for d in response["data"]: ## print d datasets.append(d["dataset"][0]["name"]) else: datasets.append(dataset) print "Datasets to import" print "\n".join(datasets) for dsetName in datasets: print "Importing %s" % dsetName files = self.getFilesFomDAS(dsetName) self.addToDataset(catalog, dsetName, files) print "Writing catalog" self.writeCatalog(catalog) print "Done"
def produceJobs(dataset, totalN, jobN): query = das_query("file dataset=%s/NANOAODSIM" % dataset) if not 'data' in query: raise Exception( 'Your das query has not worked properly - check your proxy is valid' ) files = [each['file'][0] for each in query['data']] if totalN == None: totalN = sum([f['nevents'] for f in files]) jobs = [] end_request = False counter = 0 #count number of events assigned to jobs so far for f in files: if not end_request: fileN = f['nevents'] for start in range(0, fileN, jobN): nEvents_to_end_request = totalN - counter nEvents_to_end_file = fileN - start possible_job_sizes = [ nEvents_to_end_request, nEvents_to_end_file, jobN ] job_size = min(possible_job_sizes) job = {"name": f['name'], "start": start, "jobsize": job_size} jobs.append(job) counter += job_size if counter == totalN: end_request = True break else: break return jobs
def importFromDAS(self,list_datasets): """ Import datasets from DAS to the catalog. @datasets: wildecard to be usd in dataset query """ # ret,out = commands.getstatusoutput("voms-proxy-info -e") if ret != 0: print "\n\nNo valid voms proxy found. This is needed to query DAS.\nPlease create a valid proxy running the following command:\nvoms-proxy-init -voms cms\n" sys.exit(-1) catalog = self.readCatalog() print "Importing from das %s" % list_datasets datasets = [] for dataset in list_datasets: if "*" in dataset: # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) response = das_query("dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_)) ## print response for d in response["data"]: ## print d datasets.append( d["dataset"][0]["name"] ) else: datasets.append(dataset) print "Datasets to import" print "\n".join(datasets) for dsetName in datasets: print "Importing %s" % dsetName files = self.getFilesFomDAS(dsetName) self.addToDataset(catalog,dsetName,files) print "Writing catalog" self.writeCatalog(catalog) print "Done"
action="store_true", help="Extract files from .txt file, specified above") return parser.parse_args() (opt, args) = get_options() # Create list of files files = [] if opt.doFilesFromList: listOfFiles = open(samples[opt.sample], "r") for line in listOfFiles: files.append(line[:-1]) else: das_sample = samples[opt.sample] for fdata in das_query("file dataset=%s" % das_sample, cmd="dasgoclient --dasmaps=./")['data']: files.append("root://cms-xrd-global.cern.ch/%s" % fdata['file'][0]['name']) scaleWeights = [ 'NNLO_1_1_NLO_1_1', 'NNLO_1_1_NLO_1_2', 'NNLO_1_1_NLO_1_0p5', 'NNLO_1_1_NLO_2_1', 'NNLO_1_1_NLO_2_2', 'NNLO_1_1_NLO_0p5_1', 'NNLO_1_1_NLO_0p5_0p5', 'NNLO_2_2_NLO_1_1', 'NNLO_2_2_NLO_1_2', 'NNLO_2_2_NLO_1_0p5', 'NNLO_2_2_NLO_2_1', 'NNLO_2_2_NLO_2_2', 'NNLO_2_2_NLO_0p5_1', 'NNLO_2_2_NLO_0p5_0p5', 'NNLO_0p5_0p5_NLO_1_1', 'NNLO_0p5_0p5_NLO_1_2', 'NNLO_0p5_0p5_NLO_1_0p5', 'NNLO_0p5_0p5_NLO_2_1', 'NNLO_0p5_0p5_NLO_2_2', 'NNLO_0p5_0p5_NLO_0p5_1', 'NNLO_0p5_0p5_NLO_0p5_0p5' ] # Define histograms
for gen in opt.generators.split(","): for year in opt.years.split(","): proc = "%s_%s_%s%s"%(opt.productionMode,year,gen,opt.ext) das_sample = samples[proc] if das_sample == "NA": print " --> (%s,%s,%s): Sample not available"%(opt.productionMode,year,gen) # Add column to table for pm in pmSplit[opt.productionMode]: if gen == "mg": sampleColumns = 'amc@NLO (%s)'%year else: sampleColumns = 'POWHEG (%s)'%year for b in bins[pm]: sampleColumns += " & -" filesout[pm].write(" %s \\\\ \\hline\n"%sampleColumns) else: # Create list of files from datasample files = [] for fdata in das_query("file dataset=%s"%das_sample, cmd="dasgoclient --dasmaps=./")['data']: files.append("root://cms-xrd-global.cern.ch/%s"%fdata['file'][0]['name']) # Create list of dataframes frs = [] _vars = ['genWeight','HTXS_stage_0','HTXS_stage1_1_cat_pTjet30GeV','HTXS_Higgs_pt','HTXS_njets30'] for fname in files: f_upr = upr.open(fname) t = f_upr['Events'] frs.append( t.pandas.df(_vars) ) fr = pd.concat( frs, sort=False ) # Add 1.2 bins fr['HTXS_stage1_2_cat_pTjet30GeV'] = fr.apply(lambda x: convert_to_1p2(x['HTXS_stage1_1_cat_pTjet30GeV'],x['HTXS_Higgs_pt']), axis=1) # If production mode is ggH, add NNLOPS reweighting if(opt.doNNLOPS)&(opt.productionMode == 'ggH'): if gen == "mg": fr['genWeight_NNLOPS'] = fr.apply(lambda x: NNLOPS_rwgt(NNLOPSWeights_amcatnlo,x['genWeight'],x['HTXS_njets30'],x['HTXS_Higgs_pt']), axis=1) elif gen == "powheg": fr['genWeight_NNLOPS'] = fr.apply(lambda x: NNLOPS_rwgt(NNLOPSWeights_powheg,x['genWeight'],x['HTXS_njets30'],x['HTXS_Higgs_pt']), axis=1)
def customize(self,process): self.parse() # keep useParent and secondaryDataset as exclusive options for the moment if self.options.useParentDataset and self.options.secondaryDataset != "": raise Exception("useParentDataset cannot be set together with a secondaryDataset") isFwlite = False hasOutput = False hasTFile = False sp_unused = "" if hasattr(process,"fwliteInput"): isFwlite = True if not isFwlite: hasOutput = hasattr(process,"out") hasTFile = hasattr(process,"TFileService") if hasOutput and hasTFile: tfile = self.outputFile.replace(".root","_histos.root") else: tfile = self.outputFile if self.dryRun: import sys if self.dataset and self.dataset != "": name,xsec,totEvents,files,maxEvents,sp_unused = self.dataset if self.getMaxJobs: print "maxJobs:%d" % ( min(len(files),self.nJobs) ) if len(files) != 0: if isFwlite: print "hadd:%s" % self.outputFile else: if hasOutput: print "edm:%s" % self.outputFile if hasTFile or self.tfileOut: print "hadd:%s" % tfile ## sys.exit(0) else: sys.exit(1) files = self.inputFiles if self.dataset and self.dataset != "": dsetname,xsec,totEvents,files,maxEvents,sp_unused = self.dataset if type(xsec) == float or xsec == None: print print "Error: cross section not found for dataset %s" % dsetname print putarget = None samplepu = None if self.puTarget != "": putarget = map(float, self.puTarget.split(",")) processId = self.getProcessId(dsetname) self.processId = processId #---------- if self.options.processIndex != None: self.processIndex = self.options.processIndex else: # not specified on the command line, try to take it # from the cross section file, otherwise use smallest int32 as default value # in order not to confuse it with data (index 0) if isinstance(xsec, dict): self.processIndex = xsec.get('itype', -0x7FFFFFFF) else: # note that in some cases (process not defined in cross_sections.json ?) # this can still be a float self.processIndex = -0x7FFFFFFF #---------- if isinstance(xsec, dict) and "itype" in xsec: for name,obj in process.__dict__.iteritems(): if hasattr(obj, "sampleIndex"): obj.sampleIndex = xsec["itype"] isdata = self.processType == "data" if isdata or self.targetLumi > 0. or putarget: ## look for analyzers which have lumiWeight as attribute for name,obj in process.__dict__.iteritems(): if hasattr(obj,"lumiWeight"): if isdata: obj.lumiWeight = 1. else: wei = xsec["xs"]/float(totEvents)*self.targetLumi wei *= xsec.get("br",1.) wei *= xsec.get("kf",1.) obj.lumiWeight = wei if hasattr(obj,"intLumi"): if isdata: obj.intLumi= 0 # should not be used in final fits. # setting to 0 will cause error if someone tries to use #it for normalization downsteram else: obj.intLumi=self.targetLumi if putarget and not isdata: puObj = None if hasattr(obj,"puReWeight"): puObj = obj elif hasattr(obj,"globalVariables") and hasattr(obj.globalVariables,"puReWeight"): puObj = obj.globalVariables if puObj: if not samplepu: # print dsetname # print self.pu_distribs.keys() # hack2017 = True # found_hack2017 = False # if hack2017: # if self.options.PUyear=="2017": # print dsetname.split("/")[1] # # print self.pu_distribs.keys() # print self.pu_distribs_hack_2017.keys() # # matches = filter(lambda x: x == dsetname.split("/")[1],self.pu_distribs.keys()) # matches = filter(lambda x: x == dsetname.split("/")[1],self.pu_distribs_hack_2017.keys()) # if len(matches) == 1: # found_hack2017 = True # print "FOUND HACK2017 PILEUP DISTRIBUTION WITH KEY:",matches[0] # if not found_hack2017: matches = filter(lambda x: x in dsetname, self.pu_distribs.keys() ) print matches if len(matches) > 1: print "Multiple matches, check if they're all the same" allsame = True for i in range(1,len(matches)): if self.pu_distribs[matches[0]] != self.pu_distribs[matches[i]]: allsame = False if allsame: print "They're all the same so we just take the 0th one:",matches[0] matches = [matches[0]] else: print "Not all the same... so we return to the old behavior and take an exact match, otherwise leave empty..." matches = filter(lambda x: x == dsetname, matches) if len(matches) != 1: raise Exception("Could not determine sample pu distribution for reweighting. Possible matches are [%s]. Selected [%s]\n dataset: %s" % ( ",".join(self.pu_distribs.keys()), ",".join(matches), dsetname ) ) # if self.options.PUyear=="2017": samplepu = self.pu_distribs_hack_2017[matches[0]] # else : samplepu = self.pu_distribs[matches[0]] puObj.puReWeight = True puObj.puBins = cms.vdouble( map(float, samplepu.probFunctionVariable) ) puObj.mcPu = samplepu.probValue puObj.dataPu = cms.vdouble(putarget) puObj.useTruePu = cms.bool(True) for name,obj in process.__dict__.iteritems(): if hasattr(obj,"processId"): obj.processId = str(processId) for name,obj in process.__dict__.iteritems(): if hasattr(obj,"processIndex"): obj.processIndex = int(self.processIndex) lumisToSkip = None if isdata: lumisToSkip = self.samplesMan.getLumisToSkip(dsetname) process.source.lumisToSkip = lumisToSkip.getVLuminosityBlockRange() if isdata and self.lumiMask != "": if isFwlite: sys.exit("Lumi mask not supported in FWlite",-1) import FWCore.PythonUtilities.LumiList as LumiList target = LumiList.LumiList(filename = self.lumiMask) if lumisToSkip: target = target.__sub__(lumisToSkip) process.source.lumisToProcess = target.getVLuminosityBlockRange() print process.source.lumisToProcess flist = [] sflist = [] # get the runs and lumis contained in each file of the secondary dataset if self.options.secondaryDataset: secondary_files = [fdata['file'][0]['name'] for fdata in das_query("file dataset=%s instance=prod/phys03" % self.options.secondaryDataset, cmd='dasgoclient --dasmaps=./')['data']] runs_and_lumis = {} for s in secondary_files: runs_and_lumis[str(s)] = {data['lumi'][0]['run_number'] : data['lumi'][0]['lumi_section_num'] for data in das_query("lumi file=%s instance=prod/phys03" % s, cmd='dasgoclient --dasmaps=./')['data']} for f in files: if len(f.split(":",1))>1: flist.append(str(f)) else: flist.append(str("%s%s" % (self.filePrepend,f))) # keep useParent and secondaryDataset as exclusive options for the moment if self.options.useParentDataset: parent_files = das_query("parent file=%s instance=prod/phys03" % f, cmd='dasgoclient --dasmaps=./')['data'] for parent_f in parent_files: parent_f_name = str(parent_f['parent'][0]['name']) sflist.append('root://cms-xrd-global.cern.ch/'+parent_f_name if 'root://' not in parent_f_name else parent_f_name) elif self.options.secondaryDataset != "": # match primary file to the corresponding secondary file(s) f_runs_and_lumis = {data['lumi'][0]['run_number'] : data['lumi'][0]['lumi_section_num'] for data in das_query("lumi file=%s instance=prod/phys03" % f, cmd='dasgoclient --dasmaps=./')['data']} for s_name, s_runs_and_lumis in runs_and_lumis.items(): matched_runs = set(f_runs_and_lumis.keys()).intersection(s_runs_and_lumis.keys()) for run in matched_runs: if any(lumi in f_runs_and_lumis[run] for lumi in s_runs_and_lumis[run]): sflist.append(s_name) ## mitigate server glitches by copying the input files (microAOD) on the worker node if self.copyInputMicroAOD and not self.dryRun: commands.getstatusoutput('mkdir -p input_files/') for i,f in enumerate(flist): status, out = commands.getstatusoutput('xrdcp %s ./input_files/'%f) print(out) flocal = 'file:./input_files/'+f.split('/')[-1] flist[i] = flocal if len(flist) > 0: ## fwlite if isFwlite: ## process.fwliteInput.fileNames.extend([ str("%s%s" % (self.filePrepend,f)) for f in files]) process.fwliteInput.fileNames = flist ## full framework else: ## process.source.fileNames.extend([ str("%s%s" % (self.filePrepend,f)) for f in files]) process.source.fileNames = flist if len(sflist) > 0: process.source.secondaryFileNames = cms.untracked.vstring(sflist) ## fwlite if isFwlite: process.fwliteInput.maxEvents = self.options.maxEvents process.fwliteOutput.fileName = self.outputFile ## full framework else: process.maxEvents.input = self.options.maxEvents if hasOutput: process.out.fileName = self.outputFile if hasTFile: process.TFileService.fileName = tfile if self.tfileOut: if hasTFile: print "Could not run with both TFileService and custom tfileOut" sys.exit(-1) name,attr = self.tfileOut setattr( getattr( process, name ), attr, tfile ) if self.dumpPython != "": from gzip import open pyout = open("%s.gz" % self.dumpPython,"w+") pyout.write( process.dumpPython() ) pyout.close()