def compile_histpairs(chunk_index, chunk_size, config_dir, dqmSource,
subsystem, data_series, data_sample, data_run, data_path,
ref_series, ref_sample, ref_runs, ref_paths):
config = cfg.get_subsystem(config_dir, subsystem)
# Histogram details
conf_list = config["hists"]
main_gdir = config["main_gdir"]
def_comparators = config['comparators'] if 'comparators' in config.keys(
) else None
# ROOT files
data_file = uproot.open(data_path)
ref_files = [uproot.open(ref_path) for ref_path in ref_paths]
histPairs = []
missing_data_dirs = []
missing_ref_dirs = []
for hconf in conf_list:
# Set comparators if there are none
if not 'comparators' in hconf.keys():
hconf['comparators'] = def_comparators
# Get name of hist in root file
h = str(hconf["path"].split("/")[-1])
# Get parent directory of hist
gdir = str(hconf["path"].split(h)[0])
data_dirname = "{0}{1}".format(main_gdir.format(data_run), gdir)
ref_dirnames = [
"{0}{1}".format(main_gdir.format(ref_run), gdir)
for ref_run in ref_runs
]
try:
data_dir = data_file[data_dirname[:-1]]
except:
missing_data_dirs.append(data_dirname)
continue
# raise error("Subsystem dir {0} not found in data root file".format(data_dirname))
ref_dirs = []
for iRef in range(len(ref_runs)):
try:
ref_dirs.append(ref_files[iRef][ref_dirnames[iRef][:-1]])
except:
missing_ref_dirs.append(ref_dirname)
continue
# raise error("Subsystem dir {0} not found in ref root file".format(ref_dirnames[iRef]))
data_keys = data_dir.keys()
ref_keyss = [ref_dir.keys() for ref_dir in ref_dirs]
valid_names = []
# Add existing histograms that match h
if "*" not in h:
if h in [str(keys)[0:-2] for keys in data_keys] and all(
[(h in [str(keys)[0:-2] for keys in ref_keys])
for ref_keys in ref_keyss]):
try:
data_hist = data_dir[h]
ref_hists = [ref_dir[h] for ref_dir in ref_dirs]
except Exception as e:
continue
data_hist_conc, ref_hists_conc = None, None
if 'concatenate' in hconf.keys():
try:
data_hist_conc = [
data_dir[dhc] for dhc in hconf['concatenate']
]
ref_hists_conc = [[
ref_dir[rhc] for rhc in hconf['concatenate']
] for ref_dir in ref_dirs]
except Exception as e:
continue
hPair = HistPair(dqmSource,
hconf, data_series, data_sample, data_run,
str(h), data_hist, ref_series, ref_sample,
ref_runs, str(h), ref_hists, data_hist_conc,
ref_hists_conc)
histPairs.append(hPair)
else:
# Check entire directory for files matching wildcard (Throw out wildcards with / in them as they are not plottable)
for name in data_keys:
if h.split("*")[0] in str(name) and all(
[name in ref_keys
for ref_keys in ref_keyss]) and not "<" in str(name):
if ("/" not in name[:-2]):
try:
data_hist = data_dir[name[:-2]]
ref_hists = [
ref_dir[name[:-2]] for ref_dir in ref_dirs
]
except Exception as e:
continue
hPair = HistPair(dqmSource, hconf,
data_series, data_sample, data_run,
str(name[:-2]), data_hist,
ref_series, ref_sample, ref_runs,
str(name[:-2]), ref_hists)
histPairs.append(hPair)
## TODO: "raise warning" is not an actual function, but need some way to alert
## TODO: users that histograms in json config file are missing from ROOT files - AWB 2022.06.11
# if len(missing_data_dirs) > 0:
# raise warning("The folloing subsystem dirs not found in data root file")
# for missing_data_dir in missing_data_dirs:
# raise warning("{0}".format(missing_data_dir))
# if len(missing_ref_dirs) > 0:
# raise warning("The folloing subsystem dirs not found in ref root file")
# for missing_ref_dir in missing_ref_dirs:
# raise warning("{0}".format(missing_ref_dirs))
#Return histpairs that match the chunk_index <<CAN BE IMPROVED IN THE FUTURE TO BE MORE EFFICIENT>>
return histPairs[min(chunk_index, len(histPairs)
):min(chunk_index + chunk_size, len(histPairs))]
def compile_histpairs(config_dir, subsystem, data_series, data_sample,
data_run, data_path, ref_series, ref_sample, ref_run,
ref_path):
config = cfg.get_subsystem(config_dir, subsystem)
# Histogram details
conf_list = config["hists"]
main_gdir = config["main_gdir"]
# ROOT files
data_file = ROOT.TFile.Open(data_path)
ref_file = ROOT.TFile.Open(ref_path)
histPairs = []
for hconf in conf_list:
# Get name of hist in root file
h = str(hconf["path"].split("/")[-1])
# Get parent directory of hist
gdir = str(hconf["path"].split(h)[0])
data_dirname = "{0}{1}".format(main_gdir.format(data_run), gdir)
ref_dirname = "{0}{1}".format(main_gdir.format(ref_run), gdir)
data_dir = data_file.GetDirectory(data_dirname)
ref_dir = ref_file.GetDirectory(ref_dirname)
if not data_dir:
raise error("Subsystem dir {0} not found in data root file".format(
data_dirname))
if not ref_dir:
raise error("Subsystem dir {0} not found in ref root file".format(
ref_dirname))
data_keys = data_dir.GetListOfKeys()
ref_keys = ref_dir.GetListOfKeys()
valid_names = []
# Add existing histograms that match h to valid_names
if "*" not in h:
if data_keys.Contains(h) and ref_keys.Contains(h):
valid_names.append(h)
else:
# Check entire directory for files matching wildcard
for name in [key.GetName() for key in data_keys]:
if h.split("*")[0] in name and ref_keys.Contains(name):
valid_names.append(name)
# Load the histograms and create HistPairs
for name in valid_names:
data_hist = data_dir.Get(name)
ref_hist = ref_dir.Get(name)
# This try/catch is a dirty way of checking that this objects are something plottable
try:
data_hist.SetDirectory(0)
ref_hist.SetDirectory(0)
except:
continue
hPair = HistPair(hconf, data_series, data_sample, data_run, name,
data_hist, ref_series, ref_sample, ref_run, name,
ref_hist)
histPairs.append(hPair)
data_file.Close()
ref_file.Close()
return histPairs
def compile_histpairs(chunk_index, chunk_size, config_dir, subsystem,
data_series, data_sample, data_run, data_path,
ref_series, ref_sample, ref_run, ref_path):
config = cfg.get_subsystem(config_dir, subsystem)
# Histogram details
conf_list = config["hists"]
main_gdir = config["main_gdir"]
# ROOT files
data_file = uproot.open(data_path)
ref_file = uproot.open(ref_path)
histPairs = []
for hconf in conf_list:
# Get name of hist in root file
h = str(hconf["path"].split("/")[-1])
# Get parent directory of hist
gdir = str(hconf["path"].split(h)[0])
data_dirname = "{0}{1}".format(main_gdir.format(data_run), gdir)
ref_dirname = "{0}{1}".format(main_gdir.format(ref_run), gdir)
data_dir = data_file[data_dirname[:-1]]
ref_dir = ref_file[ref_dirname[:-1]]
if not data_dir:
raise error("Subsystem dir {0} not found in data root file".format(
data_dirname))
if not ref_dir:
raise error("Subsystem dir {0} not found in ref root file".format(
ref_dirname))
data_keys = data_dir.keys()
ref_keys = ref_dir.keys()
valid_names = []
# Add existing histograms that match h
if "*" not in h:
if h in [str(keys)[0:-2] for keys in data_keys
] and h in [str(keys)[0:-2] for keys in ref_keys]:
try:
data_hist = data_dir[h]
ref_hist = ref_dir[h]
except Exception as e:
continue
hPair = HistPair(hconf, data_series, data_sample, data_run,
str(h), data_hist, ref_series, ref_sample,
ref_run, str(h), ref_hist)
histPairs.append(hPair)
else:
# Check entire directory for files matching wildcard (Throw out wildcards with / in them as they are not plottable)
for name in data_keys:
if h.split("*")[0] in str(
name) and name in ref_keys and not "<" in str(name):
if ("/" not in name[:-2]):
try:
data_hist = data_dir[name[:-2]]
ref_hist = ref_dir[name[:-2]]
except Exception as e:
continue
hPair = HistPair(hconf, data_series, data_sample,
data_run, str(name[:-2]), data_hist,
ref_series, ref_sample, ref_run,
str(name[:-2]), ref_hist)
histPairs.append(hPair)
#Return histpairs that match the chunk_index <<CAN BE IMPROVED IN THE FUTURE TO BE MORE EFFICIENT>>
return histPairs[min(chunk_index, len(histPairs)
):min(chunk_index + chunk_size, len(histPairs))]