fileSet = { k: fileset[k] for k in fileset if not ('TTGamma' in k or 'TTbar' in k or 'DY' in k or 'ST' in k or 'W1' in k or 'W2' in k or 'W3' in k or 'W4' in k) } mcType = 'MCOther' print(fileSet.keys()) output = processor.run_uproot_job( fileSet, treename='Events', processor_instance=TTGammaProcessor(mcEventYields=mcEventYields), executor=processor.futures_executor, executor_args={ 'workers': 5, 'flatten': True }, chunksize=50000, # maxchunks=1, ) elapsed = time.time() - tstart print("Total time: %.1f seconds" % elapsed) print("Total rate: %.1f events / second" % (output['EventCount'].value / elapsed)) util.save(output, f"output{mcType}_ttgamma_condorFull_4jet.coffea") if sys.argv[1] == 'Data': output = processor.run_uproot_job(
def main(): overwrite = True small = True # load the config and the cache cfg = loadConfig() cacheName = 'singleLep_small' if small else 'singleLep' # Inputs are defined in a dictionary # dataset : list of files from samples import fileset, fileset_small, fileset_1l # histograms histograms = ["MET_pt", "N_b", "N_jet", "MT", "N_spec", "pt_spec_max", "HT", "ST"] histograms += ['mbj_max', 'mjj_max', 'mlb_min', 'mlb_max', 'mlj_min', 'mlj_max'] #histograms += ['FWMT1', 'FWMT2', 'FWMT3', 'FWMT4', 'FWMT5'] #histograms += ['S', 'S_lep'] # initialize cache cache = dir_archive(os.path.join(os.path.expandvars(cfg['caches']['base']), cfg['caches'][cacheName]), serialized=True) if not overwrite: cache.load() if cfg == cache.get('cfg') and histograms == cache.get('histograms') and cache.get('simple_output'): output = cache.get('simple_output') else: # Run the processor if small: fileset = fileset_small workers = 1 else: fileset = fileset_1l workers = 6 output = processor.run_uproot_job(fileset, treename='Events', processor_instance=exampleProcessor(), executor=processor.futures_executor, executor_args={'workers': workers, 'function_args': {'flatten': False}}, chunksize=50000, ) cache['fileset'] = fileset cache['cfg'] = cfg cache['histograms'] = histograms cache['simple_output'] = output cache.dump() # Make a few plots outdir = "./tmp_plots" if not os.path.exists(outdir): os.makedirs(outdir) for name in histograms: print (name) histogram = output[name] ax = hist.plot1d(histogram,overlay="dataset", stack=True) # make density plots because we don't care about x-sec differences ax.set_yscale('linear') ax.figure.savefig(os.path.join(outdir, "{}.pdf".format(name))) ax.clear() return output
import time tstart = time.time() workers = Factory("local", manager_host_port="localhost:9123") workers.max_workers = 1 workers.min_workers = 1 workers.python_package = wq_env_tarball with workers: output = processor.run_uproot_job( fileset, treename='Events', processor_instance=MyProcessor(), executor=processor.work_queue_executor, executor_args=work_queue_executor_args, chunksize=100000, # Change this to None for a large run: maxchunks=4, ) elapsed = time.time() - tstart print(output) # Expected output: # {'sumw': defaultdict_accumulator(<class 'float'>, {'DoubleMuon': 400224.0}), 'mass': <Hist (dataset,mass) instance at 0x7f4e02708460>} if output['sumw']['DoubleMuon'] == 400224.0: print("Output is correct.")
'savemetrics': 1, # 'xrootdconfig': { # 'chunkbytes': 1024*128, # 'limitbytes': 200 * 1024**2 # }, 'cachestrategy': 'dask-worker', 'worker_affinity': True, } chunksize = 100000 if True: tic = time.time() res = processor.run_uproot_job(filelist, 'Events', NanoTestProcessor(), processor.dask_executor, config, chunksize=chunksize, maxchunks=None) toc = time.time() print("Dask client:", client) print("Total time: %.0f" % (toc - tic)) print("Events / s / thread: {:,.0f}".format(res[1]['entries'].value / res[1]['processtime'].value)) print("Bytes / s / thread: {:,.0f}".format(res[1]['bytesread'].value / res[1]['processtime'].value)) print("Events / s: {:,.0f}".format(res[1]['entries'].value / (toc - tic))) print("Bytes / s: {:,.0f}".format(res[1]['bytesread'].value / (toc - tic))) from coffea.util import save
from os.path import join, isdir, splitext reldir = splitext(__file__)[0].replace('_', '/') outdir = join(os.getenv('FH_BASE'), "Imgs", reldir) if not isdir(outdir): os.makedirs(outdir) histos = {} print('[signal]') outputs = {} for k, ds in sigDS.items(): outputs[k] = processor.run_uproot_job( {k: ds}, treename='ffNtuplizer/ffNtuple', processor_instance=LeptonjetIsoProcessor(dphi_control=False, data_type='sig'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=True), chunksize=500000, ) print("Filling..") histos['sig'] = {} for k in outputs: histos['sig'][k] = root_filling(outputs[k], k) print('[background]') output = processor.run_uproot_job( bkgDS, treename='ffNtuplizer/ffNtuple', processor_instance=LeptonjetIsoProcessor(dphi_control=False, data_type='bkg'),
else: import batch dfk = batch.configure(nodes=8, nprocs=8) job_kwargs = { 'executor': processor.parsl_executor, 'executor_args': { 'flatten': False, 'xrootdtimeout': 30 }, } fileset = utils.ensure_local(fileset) binning = np.arange(101).astype(float) # yes, hardcoded binning output = processor.run_uproot_job(fileset, treename='metaTree/PUDistribution', processor_instance=PUHists( fileset.keys(), binning), chunksize=500000, **job_kwargs) print('DONE!') print(output) set_trace() # to do: controllare che sovrascriva il pu_mc outf = uproot.recreate(f'inputs/{args.jobid}/pu_mc.root') for key, h in output.items(): vals = h.value vals /= vals.sum() outf[key] = (vals, binning) def norm(h): scale = h.allvalues.sum() for i in range(len(h)):
"container_service_names": "dask", "dask_container_port": "8787", "should_transfer_files": "YES", "when_to_transfer_output": "ON_EXIT", "+DaskSchedulerAddress": '"129.93.183.33:8787"', }) cluster.scale(jobs=1) client = Client(cluster) #, security=sec_dask) print("Dask client: ", client) exe_args = { 'client': client, 'compression': compression, } hists = processor.run_uproot_job( filelist, treename, processor_instance=proc, executor=processor.dask_executor, #executor=processor.futures_executor, executor_args=exe_args) assert (hists['cutflow']['ZJets_pt'] == 18) assert (hists['cutflow']['ZJets_mass'] == 6) assert (hists['cutflow']['Data_pt'] == 84) assert (hists['cutflow']['Data_mass'] == 66)
for this_file in infiles: index = this_file.split("_")[1].split(".json")[0] print(this_file, index) uproot.open.defaults[ "xrootd_handler"] = uproot.source.xrootd.MultithreadedXRootDSource p = HbbProcessor(year=year, tagger='v2') args = {'savemetrics': True, 'schema': NanoAODSchema} output = processor.run_uproot_job( this_file, treename="Events", processor_instance=p, executor=processor.dask_executor, executor_args={ "client": client, "skipbadfiles": 1, "schema": processor.NanoAODSchema, "treereduction": 2, }, chunksize=100000, # maxchunks=args.max, ) outfile = '/uscms/home/cmantill/nobackup/tmp/hbb-cut-based/vbf-category/ggf-vbf-ddb2/outfiles/' + str( year) + '_dask_' + index + '.coffea' util.save(output, outfile) print("saved " + outfile)
print(sample_name) samples = { sample_name : filelist } # Class -> Object #JW_Processor_instance = JW_Processor(year,setname,corrections,xsecDY) <--on developing--> JW_Processor_instance = JW_Processor(year,setname,xsecDY,pu,corrections) ## -->Multi-node Executor result = processor.run_uproot_job( samples, #dataset "Events", # Tree name JW_Processor_instance, # Class executor=processor.futures_executor, executor_args={"schema": NanoAODSchema, "workers": 20}, #maxchunks=4, ) outname = data_sample + '.futures' #outname = 'DY_test.futures' save(result,outname) elapsed_time = time.time() - start print("Time: ",elapsed_time)
output['era_2'] += processor.column_accumulator( era[totcut & (channel_ == 2)]) return output def postprocess(self, accumulator): return accumulator if __name__ == "__main__": import pandas as pd out_ = processor.run_uproot_job( dataDS, treename='ffNtuplizer/ffNtuple', processor_instance=LeptonjetEventDrawer(data_type='data'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=False), chunksize=500000, ) df_4mu = pd.DataFrame( [ out_['run_2'].value, out_['lumi_2'].value, out_['event_2'].value, out_['era_2'].value, ], index=['run', 'lumi', 'event', 'era'], dtype='Int64', ).transpose() df_4mu.sort_values(by=['run', 'lumi', 'event'], inplace=True)
"dask_container_port": "8787", "should_transfer_files": "YES", "when_to_transfer_output": "ON_EXIT", "+DaskSchedulerAddress": '"129.93.183.33:8787"', }) cluster.scale(jobs=2) client = Client(cluster)#, security=sec_dask) #cachestrategy = 'dask-worker' exe_args = { 'client': client, #'cachestrategy': cachestrategy, #'savemetrics': True, #'worker_affinity': True if cachestrategy is not None else False, } output = processor.run_uproot_job(fileset, treename = 'Events', processor_instance = METProcessor(), executor = processor.dask_executor, executor_args = exe_args ) # Generates a 1D histogram from the data output to the 'MET' key. fill_opts are optional, to fill the graph (default is a line). hist.plot1d(output['MET'], overlay='dataset', fill_opts={'edgecolor': (0,0,0,0.3), 'alpha': 0.8}) # Easy way to print all cutflow dict values. Can just do print(output['cutflow']["KEY_NAME"]) for one. for key, value in output['cutflow'].items(): print(key, value)
except yaml.YAMLError as exc: print(exc) checkfile = open('sync_outputs/desy.csv') checklist = pd.read_csv(checkfile) print(checklist.shape) t0 = time() out = processor.run_uproot_job( fileset, treename='Events', processor_instance=SignalProcessor(sync=True, categories=['eemt'], checklist=checklist), executor=processor.futures_executor, executor_args={ 'workers': 20, 'flatten': True, "nano": True }, #chunksize=1000, #maxchunks=50, ) print(out['cutflow_sync'].items()) print(out['cutflow'].items()) lumi = np.array(out['lumi'].value, dtype=int) run = np.array(out['run'].value, dtype=int) evt = np.array(out['evt'].value, dtype=int)
return accumulator if __name__ == "__main__": import os from os.path import join, isdir, splitext from FireHydrant.Analysis.PlottingOptions import * reldir = splitext(__file__)[0].replace('_', '/') outdir = join(os.getenv('FH_BASE'), "Imgs", reldir) if not isdir(outdir): os.makedirs(outdir) out_sig2mu2e = processor.run_uproot_job( sigDS_2mu2e, treename='ffNtuplizer/ffNtuple', processor_instance=GenJetProcessor(data_type='sig-2mu2e'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=False), chunksize=500000, ) out_sig4mu = processor.run_uproot_job( sigDS_4mu, treename='ffNtuplizer/ffNtuple', processor_instance=GenJetProcessor(data_type='sig-4mu'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=False), chunksize=500000, ) import re longdecay = re.compile('^.*_lxy-300$') sampleSig = re.compile(
if __name__ == "__main__": import os from os.path import join, isdir from FireHydrant.Analysis.PlottingOptions import * outdir = join(os.getenv('FH_BASE'), "Imgs", __file__.split('.')[0]) if not isdir(outdir): os.makedirs(outdir) outputs = {} outputs['bkg'] = processor.run_uproot_job( bkgDS, treename='ffNtuplizer/ffNtuple', processor_instance=LeptonjetLeadSubleadProcessor(region='SR', data_type='bkg'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=True), chunksize=500000, ) outputs['sig-2mu2e'] = processor.run_uproot_job( filterSigDS(sigDS_2mu2e), treename='ffNtuplizer/ffNtuple', processor_instance=LeptonjetLeadSubleadProcessor( region='SR', data_type='sig-2mu2e'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=True), chunksize=500000, )
print('[{}] starting submission'.format(datetime.now().strftime("%H:%M:%S"))) final_accumulator, metrics = run_uproot_job( dataset, 'otree', load('boostedHbbProcessor.coffea'), funcx_executor, executor_args={ # 'local_path': '/hadoop/store/user/awoodard/data', # 'stageout_url': 'root://deepthought.crc.nd.edu://store/user/awoodard/data', # 'local_path': '/scratch365/awoodard/funcx', # 'stageout_url': 'file:///scratch365/awoodard/funcx', 'local_path': '/scratch/midway2/annawoodard/funcx/results', 'stageout_url': 'file:///scratch/midway2/annawoodard/funcx/results', 'endpoints': [midway_uuid], 'skipbadfiles': True, 'savemetrics': True, 'xrootdtimeout': 20, # 'poll_period': 5, 'poll_period': 30, # 'tailtimeout': 500, # 'tailretry': 90, # 'batch_size': 500, 'funcx_service_address': 'https://dev.funcx.org/api/v1' # 'funcx_service_address': 'https://funcx.org/api/v1' }, pre_executor=futures_executor, chunksize=args.chunksize, metadata_cache=metadata_cache ) returned = time.time() print(metrics)
fileslice = slice(None) with open("metadata/" + options.year + ".json") as fin: samplefiles = json.load(fin) filelist = {} for dataset, info in samplefiles.items(): if options.dataset and options.dataset not in dataset: continue files = [] for file in info['files'][fileslice]: files.append(file) filelist[dataset] = files from distributed import Client client = Client('coffea-dask.fnal.gov:8786') tstart = time.time() output = processor.run_uproot_job( filelist, treename='Events', processor_instance=processor_instance, executor=processor.dask_executor, executor_args={'client': client}, chunksize=50000, ) # Pickle is not very fast or memory efficient, will be replaced by something better soon # with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: os.system("mkdir -p hists/" + options.analysis + year) save(output, 'hists/' + options.analysis + year + '/' + dataset + '.dask') dt = time.time() - tstart print(dt)
chunk_size = 15000 if "QCD_Pt_2400to3200" in samples2process[0]: chunk_size = 1500 if "QCD_Pt_3200toInf" in samples2process[0]: chunk_size = 500 print( "Processing QCD samples from which we only need a small amount of events!" ) print("Events to be processed: ", chunk_size) output = processor.run_uproot_job( subsample_files, treename='Events', processor_instance=TrijetHistogramMaker(isMC=isMC), executor=processor.futures_executor, chunksize=chunk_size, maxchunks=1, executor_args={ 'workers': args.workers, 'flatten': False, 'status': not args.condor, "schema": HackSchema }) util.save(output, f"DataHistograms_{save_tag}_{samples2process[0]}.coffea") else: output = processor.run_uproot_job( subsample_files, treename='Events', processor_instance=TrijetHistogramMaker(isMC=isMC), executor=processor.futures_executor, chunksize=250000,
if __name__ == '__main__': tmFileset = { 'CMSSW CUETPM81': [ '/Users/chrispap/QCD/new/Autumn18.QCD_HT1000to1500_TuneCP5_13TeV-madgraphMLM-pythia8_0_RA2AnalysisTree.root', '/Users/chrispap/QCD/new/Autumn18.QCD_HT1500to2000_TuneCP5_13TeV-madgraphMLM-pythia8_0_RA2AnalysisTree.root', '/Users/chrispap/QCD/new/Autumn18.QCD_HT2000toInf_TuneCP5_13TeV-madgraphMLM-pythia8_0_RA2AnalysisTree.root', ], } tmOut = processor.run_uproot_job( tmFileset, treename="TreeMaker2/PreSelection", processor_instance=TreeMakerProcessor(), executor=processor.futures_executor, executor_args={ "schema": TreeMakerSchema, "workers": 4 }, chunksize=100000 ) numerator = tmOut["nTracksHist"].integrate('dataset', 'CMSSW RECO') denominator = tmOut["nTracksHist"].integrate('dataset', 'CMSSW GEN') # make a nice ratio plot, adjusting some font sizes plt.rcParams.update({ 'font.size': 14, 'axes.titlesize': 18, 'axes.labelsize': 18, 'xtick.labelsize': 12,
## SR CUTNAMES = dict( enumerate([ 'lj0,1 sumq0', 'dphi>pi/2', 'EGM0pt>60', 'Njets<4', 'NtightB==0', ])) out_bkg = processor.run_uproot_job( bkgDS, treename='ffNtuplizer/ffNtuple', processor_instance=CutflowProcessor(data_type='bkg', region='SR', enforceNeutral=True), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=False), chunksize=500000, ) # --- CHANNEL - 2mu2e outputs = OrderedDict() h_ = out_bkg['count'].integrate('channel', slice(1, 2)) outputs.update({k[0]: v for k, v in h_.values().items()}) df_ = pd.DataFrame(outputs) for k, n in CUTNAMES.items(): df_.rename(index={k: n}, inplace=True) YieldsDf['2mu2e-SR-OS'] = df_ # --- CHANNEL - 4mu outputs = OrderedDict()
} workers = 8 if overwrite: # create .h5 file df = pd.DataFrame(df_out) df.to_hdf('data/data_X.h5', key='df', format='table', mode='w') output = processor.run_uproot_job( fileset, treename='Events', processor_instance=WHhadProcessor(), executor=processor.futures_executor, executor_args={ 'workers': workers, 'function_args': { 'flatten': False } }, chunksize=500000, ) df_out = pd.DataFrame({ 'met': output['met'].value.flatten(), 'ht': output['ht'].value.flatten(), 'lead_jet_pt': output['lead_jet_pt'].value.flatten(), 'sublead_jet_pt': output['sublead_jet_pt'].value.flatten(),
if not overwrite: cache.load() if cfg == cache.get('cfg') and histograms == cache.get( 'histograms') and cache.get('simple_output'): output = cache.get('simple_output') else: print("I'm running now") output = processor.run_uproot_job( fileset, "Events", charge_flip_ss(year=year, variations=[], accumulator=desired_output), exe, exe_args, chunksize=500000, ) cache['fileset'] = fileset cache['cfg'] = cfg cache['histograms'] = histograms cache['simple_output'] = output cache.dump() import matplotlib.pyplot as plt import mplhep as hep plt.style.use(hep.style.CMS)
'debug-log': 'debug.log', 'transactions-log': 'tr.log', 'stats-log': 'stats.log', 'verbose': False, 'port': [9123,9130], 'environment-file': topeftenv.get_environment(), 'master-name': '{}-workqueue-coffea'.format(os.environ['USER']), 'print-stdout': True, 'skipbadfiles': False, 'schema': NanoAODSchema, 'extra-input-files': ["topeft.py"] } # Run the processor and get the output tstart = time.time() output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks) dt = time.time() - tstart print('Processed {} events in {} seconds ({:.2f} evts/sec).'.format(nevts_total,dt,nevts_total/dt)) nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) # This is taken from the DM photon analysis... # Pickle is not very fast or memory efficient, will be replaced by something better soon # with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: if not outpath.endswith('/'): outpath += '/' if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout:
def main(): overwrite = True # load the config and the cache cfg = loadConfig() # Inputs are defined in a dictionary # dataset : list of files fileset = { 'tW_scattering': glob.glob("/hadoop/cms/store/user/dspitzba/nanoAOD/ttw_samples/0p1p2/tW_scattering__nanoAOD/merged/*.root"), "TTW": glob.glob("/hadoop/cms/store/user/dspitzba/nanoAOD/ttw_samples/0p1p2/TTWJetsToLNu_TuneCP5_13TeV-amcatnloFXFX-madspin-pythia8__RunIIAutumn18NanoAODv6-Nano25Oct2019_102X_upgrade2018_realistic_v20_ext1-v1/merged/*.root") \ + glob.glob("/hadoop/cms/store/user/dspitzba/nanoAOD/ttw_samples/0p1p2/TTWJetsToQQ_TuneCP5_13TeV-amcatnloFXFX-madspin-pythia8__RunIIAutumn18NanoAODv6-Nano25Oct2019_102X_upgrade2018_realistic_v20-v1/merged/*.root"), #"ttbar": glob.glob("/hadoop/cms/store/user/dspitzba/nanoAOD/ttw_samples/0p1p2/TTJets_SingleLeptFromT_TuneCP5_13TeV-madgraphMLM-pythia8__RunIIAutumn18NanoAODv6-Nano25Oct2019_102X_upgrade2018_realistic_v20-v1/merged/*.root") # adding this is still surprisingly fast (20GB file!) } # histograms histograms = ["MET_pt", "Jet_pt", "Jet_eta", "Jet_pt_fwd", "W_pt_notFromTop", "GenJet_pt_fwd", "Spectator_pt", "Spectator_eta"] histograms+= ["Top_pt", "Top_eta", "Antitop_pt", "Antitop_eta", "W_pt", "W_eta", "N_b", "N_jet"] # initialize cache cache = dir_archive(os.path.join(os.path.expandvars(cfg['caches']['base']), cfg['caches']['simpleProcessor']), serialized=True) if not overwrite: cache.load() if cfg == cache.get('cfg') and histograms == cache.get('histograms') and fileset == cache.get('fileset') and cache.get('simple_output'): output = cache.get('simple_output') else: # Run the processor output = processor.run_uproot_job(fileset, treename='Events', processor_instance=exampleProcessor(), executor=processor.futures_executor, executor_args={'workers': 12, 'function_args': {'flatten': False}}, chunksize=500000, ) cache['fileset'] = fileset cache['cfg'] = cfg cache['histograms'] = histograms cache['simple_output'] = output cache.dump() # Make a few plots outdir = "./tmp_plots" if not os.path.exists(outdir): os.makedirs(outdir) for name in histograms: print (name) histogram = output[name] if name == 'MET_pt': # rebin new_met_bins = hist.Bin('pt', r'$E_T^{miss} \ (GeV)$', 20, 0, 200) histogram = histogram.rebin('pt', new_met_bins) if name == 'W_pt_notFromTop': # rebin new_pt_bins = hist.Bin('pt', r'$p_{T}(W) \ (GeV)$', 25, 0, 500) histogram = histogram.rebin('pt', new_pt_bins) ax = hist.plot1d(histogram,overlay="dataset", density=False, stack=True) # make density plots because we don't care about x-sec differences ax.set_yscale('linear') # can be log #ax.set_ylim(0,0.1) ax.figure.savefig(os.path.join(outdir, "{}.pdf".format(name))) ax.clear() ax = hist.plot1d(histogram,overlay="dataset", density=True, stack=False) # make density plots because we don't care about x-sec differences ax.set_yscale('linear') # can be log #ax.set_ylim(0,0.1) ax.figure.savefig(os.path.join(outdir, "{}_shape.pdf".format(name))) ax.clear() return output
def main(): # start run time clock tstart = time.time() # get options from command line parser = OptionParser() parser.add_option('-d', '--dataset', help='dataset', dest='dataset') parser.add_option('-N', '--nFiles', help='nFiles', dest='nFiles', type=int, default=-1) parser.add_option('-M', '--startFile', help='startFile', dest='startFile', type=int, default=0) parser.add_option( '--condor', help='running on condor', dest='condor', default=False, action='store_true') parser.add_option( '--dask', help='run w/ dask', dest='dask', default=False, action='store_true') parser.add_option( '--port', help='port for dask status dashboard (localhost:port)', dest='port', type=int, default=8787) parser.add_option( '--mincores', help='dask waits for min # cores', dest='mincores', type=int, default=4) parser.add_option( '--quiet', help='suppress status printouts', dest='quiet', default=False, action='store_true') parser.add_option('-w', '--workers', help='Number of workers to use for multi-worker executors (e.g. futures or condor)', dest='workers', type=int, default=8) parser.add_option('-s', '--chunksize', help='Chunk size', dest='chunksize', type=int, default=10000) parser.add_option('-m', '--maxchunks', help='Max number of chunks (for testing)', dest='maxchunks', type=int, default=None) options, args = parser.parse_args() # set output root file sample = options.dataset # getting dictionary of files from a sample collection e.g. "2016_QCD, 2016_WJets, 2016_TTJets, 2016_ZJets" fileset = s.getFileset(sample, True, options.startFile, options.nFiles) outfile = "MyAnalysis_%s_%d" % (sample, options.startFile) if options.condor or options.dask else "test" # get processor args exe_args = {'workers': options.workers, 'flatten': False} if options.dask: exe_args = use_dask(options.condor,options.workers,options.port) if options.quiet: exe_args['status'] = False client = exe_args['client'] while len(client.ncores()) < options.mincores: print('Waiting for more cores to spin up, currently there are {0} available...'.format(len(client.ncores()))) print('Dask client info ->', client) time.sleep(10) sf = s.sfGetter(sample) print("scaleFactor = {}".format(sf)) # run processor output = processor.run_uproot_job( fileset, treename='TreeMaker2/PreSelection', processor_instance=MainProcessor(sample,sf), executor=processor.dask_executor if options.dask else processor.futures_executor, executor_args=exe_args, chunksize=options.chunksize, maxchunks=options.maxchunks, ) # export the histograms to root files ## the loop makes sure we are only saving the histograms that are filled values_dict = {} branchdict = {} for v in output.keys(): if len(output[v].value) > 0: branchdict[v] = uproot.newbranch("f4") values_dict[v] = output[v].value tree = uproot.newtree(branchdict) if values_dict != {}: print("saving root files...") with uproot.recreate("{}.root".format(outfile)) as f: f["tree"] = tree f["tree"].extend(values_dict) # print run time in seconds dt = time.time() - tstart print("run time: %.2f [sec]" % (dt))
from os.path import join, isdir, splitext reldir = splitext(__file__)[0].replace('_', '/') outdir = join(os.getenv('FH_BASE'), "Imgs", reldir) if not isdir(outdir): os.makedirs(outdir) import re longdecay = re.compile('^.*_lxy-300$') # ---------------------------------------------------------- ## mu cand efficiency, resolution output = processor.run_uproot_job( sigDS_2mu2e, treename='ffNtuplizer/ffNtuple', processor_instance=MuEffiResoProcessor(), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=False), chunksize=500000, ) fig, ax = plt.subplots(figsize=(8, 6)) hist.plotratio(num=output['lxy'][longdecay].sum('dataset').integrate( 'reco', 'true'), denom=output['lxy'][longdecay].sum('dataset').integrate( 'reco', 'inclusive'), overflow='over', error_opts={ 'marker': 'o', }, ax=ax, label='PFMu+DSAMu')
#files = {'Charmonium2018AOD': filesets['Charmonium2018AOD'][:]} files = {'MonteCarlo2017AOD': filesets['MonteCarlo2017AOD'][1:2]} # creating necessary folders into dir output data os.system("mkdir -p output/" + args.name) os.system("rm -rf output/" + args.name + "/*") # If the process is for data or mc if (args.data): analysis_type = 'data' if (args.mc): analysis_type = 'mc' if config_yaml['executor'] == 'futures_executor': output = processor.run_uproot_job(files, treename='Events', processor_instance=EventSelectorProcessor(args.name, analysis_type), executor=processor.futures_executor, # Uses python futures to multiprocessing executor_args={"schema": BaseSchema, 'workers': config_yaml['n_cores']}, # BaseSchema returns a base.nano-events object chunksize=config_yaml['chunksize'], ) elif config_yaml['executor'] == 'iterative_executor': output = processor.run_uproot_job(files, treename='Events', processor_instance=EventSelectorProcessor(args.name, analysis_type), executor=processor.iterative_executor, executor_args={'schema': BaseSchema}, chunksize=config_yaml['chunksize'], ) elapsed = round(time.time() - tstart, 2) print(f"Process finished in: {elapsed} s")
for dataset, info in samplefiles.items(): filelist = {} if options.dataset and options.dataset not in dataset: continue print('Processing:', dataset) files = [] for file in info['files'][fileslice]: files.append(file) filelist[dataset] = files tstart = time.time() output = processor.run_uproot_job( filelist, treename='Events', processor_instance=processor_instance, executor=processor.futures_executor, executor_args={ 'nano': True, 'workers': options.workers }, ) #nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) #nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) #print("Filled %.1fM bins" % (nbins/1e6, )) #print("Nonzero bins: %.1f%%" % (100*nfilled/nbins, )) os.system("mkdir -p hists/" + options.processor) save(output, 'hists/' + options.processor + '/' + dataset + '.futures') dt = time.time() - tstart nworkers = options.workers print("%.2f us*cpu overall" % (1e6 * dt * nworkers, ))
if __name__ == "__main__": import os import re from os.path import join, isdir, splitext from FireHydrant.Analysis.PlottingOptions import * reldir = splitext(__file__)[0].replace('_', '/') outdir = join(os.getenv('FH_BASE'), "Imgs", reldir) if not isdir(outdir): os.makedirs(outdir) output_2mu2e = processor.run_uproot_job( sigDS_2mu2e, treename='ffNtuplizer/ffNtuple', processor_instance=LjTkIsoProcessor(data_type='sig-2mu2e'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=False), chunksize=500000, ) output_4mu = processor.run_uproot_job( sigDS_4mu, treename='ffNtuplizer/ffNtuple', processor_instance=LjTkIsoProcessor(data_type='sig-4mu'), executor=processor.futures_executor, executor_args=dict(workers=12, flatten=False), chunksize=500000, ) output_bkg = processor.run_uproot_job( bkgDS,
if not overwrite: cache.load() if cfg == cache.get('cfg') and histograms == cache.get( 'histograms') and cache.get('simple_output'): output = cache.get('simple_output') else: print("I'm running now") output = processor.run_uproot_job( fileset, "Events", trilep_analysis(year=year, variations=variations, accumulator=desired_output), exe, exe_args, chunksize=250000, ) cache['fileset'] = fileset cache['cfg'] = cfg cache['histograms'] = histograms cache['simple_output'] = output cache.dump() lines = ['entry'] lines += [ 'filter', 'lepveto',
pt=zp.p4.pt.flatten())#, #weight=df['genWeight']) print(output['hmass'].values()) return output def postprocess(self, accumulator): return accumulator samples = {"default":args.files} #with open('files_prev.json') as fin: # samples = json.load(fin) output = processor.run_uproot_job(samples, treename='Events', processor_instance=GenVisualizer(), executor=processor.futures_executor, executor_args={'workers': 4}, chunksize=500000, ) save(output, 'genstuff.coffea') output = load("genstuff.coffea") #hmass = output["hmass"] #bin_contents = hmass.values()[('ZPrimeToQQ_DMsimp_HT400_M50',)] #edges = hmass.axis('mass').edges() #edge_pairs = [(edges[i], edges[i+1]) for i in range(len(edges)-1)] #histd = zip(edge_pairs, bin_contents) #for thing in histd: # print(thing)