def get_sample_meta(fileset, samples, workers=10, skipbadfiles=True): from processor.default_accumulators import add_files_to_output, add_processes_to_output meta_accumulator = {} add_files_to_output(fileset, meta_accumulator) add_processes_to_output(fileset, meta_accumulator) meta_output = processor.run_uproot_job( fileset, "Runs", meta_processor(accumulator=meta_accumulator), processor.futures_executor, { 'workers': workers, "skipbadfiles": skipbadfiles, }, chunksize=100000, ) # now clean up the output, get skipped files per data set meta = {} for sample in fileset: meta[sample] = meta_output[sample] good_files = [] skipped_files = [] for rootfile in fileset[sample]: if meta_output[rootfile]: good_files.append(rootfile) else: skipped_files.append(rootfile) meta[sample]['good_files'] = good_files meta[sample]['n_good'] = len(good_files) meta[sample]['bad_files'] = skipped_files meta[sample]['n_bad'] = len(skipped_files) meta[sample]['xsec'] = samples[sample]['xsec'] return meta
cacheName), serialized=True) histograms = sorted(list(desired_output.keys())) year = 2018 samples = get_samples() fileset = make_fileset(['QCD'], samples, redirector=redirector_ucsd, small=True) meta = get_sample_meta(fileset, samples) add_processes_to_output(fileset, desired_output) desired_output.update({ "single_mu_fakeable": hist.Hist("Counts", dataset_axis, pt_axis, eta_axis), "single_mu": hist.Hist("Counts", dataset_axis, pt_axis, eta_axis) }) exe_args = { 'workers': 16, 'function_args': { 'flatten': False }, "schema": NanoAODSchema, "skipbadfiles": True,