def get_sample_meta(fileset, samples, workers=10, skipbadfiles=True):
from processor.default_accumulators import add_files_to_output, add_processes_to_output
meta_accumulator = {}
add_files_to_output(fileset, meta_accumulator)
add_processes_to_output(fileset, meta_accumulator)
meta_output = processor.run_uproot_job(
fileset,
"Runs",
meta_processor(accumulator=meta_accumulator),
processor.futures_executor,
{
'workers': workers,
"skipbadfiles": skipbadfiles,
},
chunksize=100000,
)
# now clean up the output, get skipped files per data set
meta = {}
for sample in fileset:
meta[sample] = meta_output[sample]
good_files = []
skipped_files = []
for rootfile in fileset[sample]:
if meta_output[rootfile]:
good_files.append(rootfile)
else:
skipped_files.append(rootfile)
meta[sample]['good_files'] = good_files
meta[sample]['n_good'] = len(good_files)
meta[sample]['bad_files'] = skipped_files
meta[sample]['n_bad'] = len(skipped_files)
meta[sample]['xsec'] = samples[sample]['xsec']
return meta
cacheName),
serialized=True)
histograms = sorted(list(desired_output.keys()))
year = 2018
samples = get_samples()
fileset = make_fileset(['QCD'],
samples,
redirector=redirector_ucsd,
small=True)
meta = get_sample_meta(fileset, samples)
add_processes_to_output(fileset, desired_output)
desired_output.update({
"single_mu_fakeable":
hist.Hist("Counts", dataset_axis, pt_axis, eta_axis),
"single_mu":
hist.Hist("Counts", dataset_axis, pt_axis, eta_axis)
})
exe_args = {
'workers': 16,
'function_args': {
'flatten': False
},
"schema": NanoAODSchema,
"skipbadfiles": True,