def compare_yoda_files(ref_file, tuned_dict_files, options=None):
tuned_dict = OrderedDict()
ref_data = yoda.read(ref_file)
for key,value in tuned_dict_files.iteritems():
tuned_dict[key] = yoda.read(value)
compare_yoda(ref_data, tuned_dict, options)
def read_histos(path):
"""
Load histograms from a YODA-supported file type, into a dict of path -> yoda.Histo[DataBin]
"""
import yoda
from packaging import version
if version.Version(yoda.__version__.decode()) < version.parse("1.8.0"):
return read_yoda_pre180(path)
histos = {}
s2s, types = [], []
aos = yoda.read(path, asdict=False)
try:
for ao in aos:
import os
if os.path.basename(ao.path()).startswith("_"): continue
if "/RAW/" in ao.path(): continue
types.append(ao.type())
s2s.append(ao.mkScatter())
del aos
for s2, tp in zip(s2s, types):
if s2.dim() != 2: continue
bins = [
(p.xMin(), p.xMax(), p.y(), p.yErrAvg()) for p in s2.points()
] # This stores the bin heights as y-values
histos[s2.path()] = bins
del s2s
except Exception as e:
print("read_histos --- Can't load histos from file '%s': %s" %
(path, e))
return histos
def read_histos(path):
"Load histograms from a YODA-supported file type, into a dict of path -> yoda.Histo[DataBin]"
histos = {}
try:
import yoda
s2s = []
types = []
aos = yoda.read(path, asdict=False)
for ao in aos:
import os
## Skip the Rivet cross-section and event counter objects
# TODO: Avoid Rivet-specific behaviour by try block handling & scatter.dim requirements
if os.path.basename(ao.path).startswith("_"): continue
if "/RAW/" in ao.path: continue
##
types.append(ao.type)
s2s.append(ao.mkScatter())
del aos #< pro-active YODA memory clean-up
for s2, tp in zip(
s2s, types
): # filter(lambda x:x.dim==2, s2s): # Filter for Scatter1D
if s2.dim != 2: continue
bins = [(p.xMin, p.xMax, p.y, p.yErrAvg) for p in s2.points
] # This stores the bin heights as y-values
histos[s2.path] = bins
del s2s #< pro-active YODA memory clean-up
except Exception as e:
print("Can't load histos from file '%s': %s" % (path, e))
return histos
def readYODA(file=None, key=None):
"""
Read histogram from yoda file
:param file: path to yoda file.
:type args: str
:param key: histogram name.
:type key: str
:param alias: histogram alias used inside FP
"""
if not file or not key:
raise Exception("file and key are mandatory arguments, please specify both")
h_yoda = yoda.read(file)[key]
if isinstance(h_yoda, yoda.Histo1D):
#---ROOT histogram
h_root = ROOT.TH1F(key, "", len(h_yoda.xMins()), np.append(h_yoda.xMins(), h_yoda.xMaxs()[-1]))
#---fill visible bins
for i, v in enumerate(h_yoda.yVals()):
h_root.SetBinContent(i+1, v)
h_root.SetBinError(i+1, h_yoda.yErrs()[i])
#---fill overflow and underflow
h_root.SetBinContent(0, h_yoda.underflow.sumW)
h_root.SetBinError(0, h_yoda.underflow.errW)
h_root.SetBinContent(h_root.GetNbinsX()+1, h_yoda.overflow.sumW)
h_root.SetBinError(h_root.GetNbinsX()+1, h_yoda.overflow.errW)
else:
raise Exception("Histogram type not supported: only Histo1D are currently supported")
return h_root
def getRivetRefData(anas=None):
"Find all Rivet reference data files"
refhistos = {}
for var in ("RIVET_ANALYSIS_PATH", "RIVET_DATA_PATH", "RIVET_REF_PATH",
"RIVET_INFO_PATH", "RIVET_PLOT_PATH"):
if var in os.environ:
abspaths = map(os.path.abspath, os.environ[var].split(":"))
os.environ[var] = ":".join(abspaths)
rivet_data_dirs = rivet.getAnalysisRefPaths()
dirlist = []
for d in rivet_data_dirs:
import glob
if anas is None:
dirlist.append(glob.glob(os.path.join(d, '*.yoda')))
else:
for a in anas:
dirlist.append(glob.glob(os.path.join(d, a + '*.yoda')))
for filelist in dirlist:
for infile in filelist:
analysisobjects = yoda.read(infile)
for path, ao in analysisobjects.iteritems():
if not any(x in path for x in ['d01', 'd02', 'd03', 'd04']):
continue
aop = rivet.AOPath(ao.path)
if aop.isref():
ao.path = aop.basepath(keepref=False)
refhistos[ao.path] = ao
return refhistos
def readInputJason(self, json_file):
data = json.load(open(json_file))
self.tune = TuneMngr(data['parameters'])
# self.nRuns = str_to_int(data['nRuns'])
self.nRuns = self.tune.generate()[0]
print "parameters input:", data['parameters']
print "total runs:", self.nRuns
print "minimum for Professor:", self.tune.minimum_runs_for_Prof()
if self.nRuns < self.tune.minimum_runs_for_Prof():
print "!!! Increase the number of parameters' values !!!"
self.nEventsPerRun = str_to_int(data['nEventsPerRun'])
self.seed = data['seed']
self.nEventsPerJob = str_to_int(data['nEventsPerJob'])
self.nJobsPerRun = int(self.nEventsPerRun / self.nEventsPerJob)
self.anaID = data['rivet_analysis']
self.process = data['pythia_process']
self.pythia_opt = data.get('pythia_options', None)
self.js = json_file
self.queue_name = data.get('queue', 'shared')
self.time = data.get('time', '1:00:00')
self.repo = data.get('repo', 'm1029')
default_cfg = "/global/homes/x/xju/code/MCTuning/pythia/data/atlas_detector_cfg.yoda"
detector_cfg = data.get("detector_cfg", default_cfg)
print "reading detector configuration: ", detector_cfg
self.detector_hist2D = yoda.read(detector_cfg)
#self.tune.update_nickname(self.detector_hist2D)
return True
def read_hist_from_yoda(self, filename, name):
yodafile = yoda.read(filename)
try:
self.hist = yodafile[name]
except:
print "error: \"%s\" is not in \"%s\"" % (name, filename)
exit(1)
def getHistos(filelist):
"""Loop over all input files. Only use the first occurrence of any REF-histogram
and the first occurrence in each MC file for every MC-histogram."""
# return reference histograms,
# mchistos = mc plots,
# xsec = generated xsec and its uncertainty,
# Nev = sum of weights, sum of squared weight, number of events
# (derived from rivet-cmphistos)
mchistos = {}
xsec = {}
Nev = {}
# for infile in filelist:
mchistos.setdefault(filelist, {})
analysisobjects = yoda.read(filelist)
print len(analysisobjects), "analysisobjects in", filelist
for path, ao in analysisobjects.iteritems():
if path.startswith('/_EVTCOUNT'):
Nev = ao
if path.startswith('/_XSEC'):
xsec = ao
# Conventionally don't plot data objects whose names start with an
# underscore
if os.path.basename(path).startswith("_"):
continue
if path.startswith('/REF/'):
# Reference histograms are read elsewhere.
continue
else:
if path not in mchistos[filelist]:
mchistos[filelist][path] = ao
return mchistos, xsec, Nev
def do_binned_plots_per_region_angle(setup, rivet_files, rivet_labels):
"""Do individual binned plots"""
region = setup.region
unfolder = region['unfolder']
hbc = unfolder.hist_bin_chopper
# Iterate through pt bins - gen binning
print("Doing YodaGenPtBinnedPlotter...")
pt_bins = unfolder.binning_handler.get_pt_bins(binning_scheme='generator',
is_signal_region=True)
gen_pt_binned_plotter = YodaGenPtBinnedPlotter(setup=setup,
bins=pt_bins,
hist_bin_chopper=hbc,
unfolder=unfolder)
# Add RIVET & style dicts
for rf, rl in zip(rivet_files, rivet_labels):
yoda_dict = yoda.read(rf)
gen_pt_binned_plotter.add_rivet_entry(
yoda_dict, style_dict=SAMPLE_STYLE_DICTS[dataframe_yoda_key(rl)])
gen_pt_binned_plotter.plot_unfolded_with_yoda_normalised(do_chi2=True,
do_zoomed=True)
return hbc
def init_ref():
"""Function to load all reference data and theory *.yoda data"""
refFiles = []
global scaledYet
print "Gathering all reference Data (and Theory, if available)"
rivet_data_dirs = rivet.getAnalysisRefPaths()
for dirs in rivet_data_dirs:
import glob
refFiles.append(glob.glob(os.path.join(dirs, '*.yoda')))
for fileList in refFiles:
for f in fileList:
aos = yoda.read(f)
for path, ao in aos.iteritems():
if ao.type != "Scatter2D":
ao = yoda.mkScatter(ao)
if ao.type == "Scatter1D":
ao = util.mkScatter2D(ao)
if path.startswith('/REF/'):
refObj[path] = ao
scaledYet[path] = False
if path.startswith('/THY/'):
refObj[path] = ao
scaledYet[path] = False
global REFLOAD
REFLOAD = True
def getRivetRefData(anas=None):
"Find all Rivet reference data files"
refhistos = {}
for var in ("RIVET_ANALYSIS_PATH", "RIVET_DATA_PATH", "RIVET_REF_PATH", "RIVET_INFO_PATH", "RIVET_PLOT_PATH"):
if var in os.environ:
abspaths = map(os.path.abspath, os.environ[var].split(":"))
os.environ[var] = ":".join(abspaths)
rivet_data_dirs = rivet.getAnalysisRefPaths()
dirlist = [ ]
for d in rivet_data_dirs:
import glob
if anas is None:
dirlist.append(glob.glob(os.path.join(d, '*.yoda')))
else:
for a in anas:
dirlist.append(glob.glob(os.path.join(d, a+'*.yoda')))
for filelist in dirlist:
for infile in filelist:
analysisobjects = yoda.read(infile)
for path, ao in analysisobjects.iteritems():
if not any(x in path for x in ['d01', 'd02', 'd03', 'd04']): continue
aop = rivet.AOPath(ao.path)
if aop.isref():
ao.path = aop.basepath(keepref=False)
refhistos[ao.path] = ao
return refhistos
def ATLAS_2017_I1519428(in_file, out_file):
"""
Downloaded table:
1. change indexes [0-8] to [1-9]
2. provide another output that contains data only i.e. y01
3. divide luminosity, lumi = 37 fb^{-1} = 37 x 10^{12} mb^{-1}.
"""
data = yoda.read(in_file)
lumi = 3.7e4 # in unit of 1/pb
lumi_err = 0.032*lumi
new_rivet = []
new_ref = []
for key, value in data.iteritems():
# change Key
index = int(re.search('d0(.+?)-x', key).group(1))
new_key = key.replace('d0{}'.format(index), 'd0{}'.format(index+1))
value.setAnnotation('Path', new_key)
if index < 2:
# scale to cross section of di-jet distributions
# scale each bin only for
for point in value.points:
bin_width = point.xErrs.minus + point.xErrs.plus
if abs(point.yErrs.minus) < 1E-5:
# Use poisson error for observed events
y_err = math.sqrt(point.y)
# if y-axis is zero, err = 0
# else: error propagation
if abs(point.y) < 1E-5:
xs_err = 0.
else:
xs_err = point.y/lumi * math.sqrt(1/point.y + lumi_err**2/lumi**2)
xs_err /= bin_width
point.yErrs = (xs_err, xs_err)
else:
point.yErrs = (point.yErrs.minus/lumi, point.yErrs.plus/lumi)
point.y = point.y / lumi / bin_width
else:
pass
# scale bin_width for chi2 distributions
#for p in value.points:
# bin_width = p.xErrs.minus + p.xErrs.plus
# p.y = p.y / bin_width
new_rivet.append(value)
if "y01" in key:
new_ref.append(value)
yoda.write(new_rivet, out_file)
yoda.write(new_ref, "ATLAS_2017_I1519428_ref.yoda")
def main(arg):
infile = arg
outfile = arg[:-5] + '_scatter.yoda'
aos = yoda.read(infile)
scatter = []
hs = [h for h in aos.values()]
for h in hs:
s = h.mkScatter()
scatter.append(s)
yoda.write(scatter, outfile)
sys.exit(0)
def yoda_to_pickle(in_fname, out_fname, record=None):
info("py2_yoda_interface()",
"using {} as input .yoda file".format(in_fname))
info("py2_yoda_interface()",
"using {} as output pickle binary".format(out_fname))
import_yoda()
yoda_file = yoda.read(in_fname)
pickle.dump(yoda_to_dict(yoda_file), open(out_fname, "wb"))
if record is None: return
info("py2_yoda_interface()", "storing record in file {}".format(record))
rec_file = open(record, "w")
rec_file.write(in_fname + "\n")
rec_file.write(out_fname + "\n")
rec_file.close()
def getSRs(filename):
sr1 = 0
sr2 = 0
sr3 = 0
sr4 = 0
sr5 = 0
sr6 = 0
sr7 = 0
counters = yoda.read(filename, asdict=False)
sr1 = counters[6].val
sr2 = counters[7].val
sr3 = counters[8].val
sr4 = counters[9].val
sr5 = counters[10].val
sr6 = counters[11].val
sr7 = counters[12].val
return [sr1, sr2, sr3, sr4, sr5, sr6, sr7]
def getRivetRefData(refhistos, anas=None):
"Find all Rivet reference data files"
rivet_data_dirs = rivet.getAnalysisRefPaths()
dirlist = []
for d in rivet_data_dirs:
if anas is None:
import glob
dirlist.append(glob.glob(os.path.join(d, '*.yoda')))
else:
dirlist.append([os.path.join(d, a + '.yoda') for a in anas])
for filelist in dirlist:
# TODO: delegate to getHistos?
for infile in filelist:
analysisobjects = yoda.read(infile)
for path, ao in analysisobjects.iteritems():
if path.startswith('/REF/'):
if path not in refhistos:
refhistos[path] = ao
def convert_yoda_to_json(yoda_path, json_path):
with open(json_path, 'w+') as f:
json_serializable = {}
for k, v in yoda.read(yoda_path).items():
if type(v) == yoda.core.Histo1D:
json_serializable[k] = [b.sumW for b in v.bins]
elif type(v) == yoda.core.Scatter1D:
json_serializable[k] = [p.x for p in v.points]
elif type(v) == yoda.core.Scatter2D:
json_serializable[k] = [(p.x, p.y) for p in v.points]
elif type(v) == yoda.core.Profile1D:
json_serializable[k] = [b.sumWY for b in v.bins]
elif type(v) == yoda.core.Counter:
json_serializable[k] = v.val
else:
print(
'WARNING: yoda object of type {} encountered (no known conversion)! Object is being ignored...'
.format(type(v)))
json.dump(json_serializable, f, sort_keys=True)
def ATLAS_2014_I1268975(in_file, out_file):
data = yoda.read(in_file)
new_data = []
pre_fix = '/REF/ATLAS_2014_I1268975'
for i in range(2):
# two jet alg. atkt4 and atkt6
for j in range(6):
# six y* bins
index = "{}/d{:0=2}-x01-y0{}".format(pre_fix, i+1, j+1)
ref_index_number = i*6 + j + 1
ref_index = "{}/d{:0=2}-x01-y01".format(pre_fix, ref_index_number)
try:
scatter2D = data[ref_index]
except KeyError:
print(ref_index,"not found")
continue
scatter2D.setAnnotation('Path', index)
new_data.append(scatter2D)
yoda.write(new_data, out_file)
def init_ref():
"""Function to load all reference data into memory
Here reference data can be either experimental data or theoretical prediction"""
refFiles = []
#refObj = {}
print "Gathering all reference Data"
rivet_data_dirs = rivet.getAnalysisRefPaths()
for dirs in rivet_data_dirs:
import glob
refFiles.append(glob.glob(os.path.join(dirs, '*.yoda')))
for fileList in refFiles:
for f in fileList:
aos = yoda.read(f)
for path, ao in aos.iteritems():
if path.startswith('/REF/'):
refObj[path] = ao
if path.startswith('/THY/'):
#TODO provide this information in the first place!
refObj[path] = ao
global REFLOAD
REFLOAD = True
def main(argv):
args = parseArgs(argv)
for run in range(args.num):
# Read input files
aos = yoda.read(args.dir + os.path.sep + str(run).zfill(4) +
os.path.sep + args.yodafile)
fpar = open(
args.dir + os.path.sep + str(run).zfill(4) + os.path.sep +
args.paramfile, 'r')
param = {}
for line in fpar:
(k, v) = line.split()
param[k.strip()] = v.strip()
fpar.close()
# Extract scatterplots from yoda file
scatters = []
for aopath, ao in aos.iteritems():
scatterplot = ao.mkScatter()
scatterplot.setAnnotation("Run_Directory", args.dir)
for key, value in param.iteritems():
scatterplot.setAnnotation("Tune_Parameter_" + key, value)
# Check for patterns and unpatterns
save = True
if args.patterns != None:
save = False
for pattern in args.patterns:
if pattern in scatterplot.path:
save = True
if args.unpatterns != None:
for unpattern in args.unpatterns:
if unpattern in scatterplot.path:
save = False
if save:
scatters.append((aopath, scatterplot))
yoda.writeYODA(dict(scatters),
"run" + str(run).zfill(4) + "_" + args.yodafile)
def patch_yodaref(yoda_from_hepdata):
"""\
Take a YODA file and check if the reference data contained in the file is in need of post-processing.
If so, apply relevant post-processing steps and return.
:param yoda_from_hepdata: YODA file containing reference data from HEPData for post-processing
:return: YODA file with post-processed data from HEPData
"""
import importlib, yoda
import rivet.hepdatapatches as hdpatch
hepdata_content = yoda.read(yoda_from_hepdata)
for tag in hepdata_content:
if tag.startswith("/REF"):
routine, tableid = tag.rstrip("/")[5:].split('/')
if hasattr(hdpatch, routine):
# get relevant patch function for this routine and apply patch
routine_patcher = importlib.import_module(
"rivet.hepdatapatches." + routine)
hepdata_content[tag] = routine_patcher.patch(
tag, hepdata_content[tag])
return hepdata_content
def read_histos_yoda(path):
"Load histograms from a YODA-supported file type, into a dict of path -> yoda.Histo[DataBin]"
histos = {}
try:
import yoda
s2s = []
aos = yoda.read(path, asdict=False)
for ao in aos:
import os
## Skip the Rivet cross-section and event counter objects
# TODO: Avoid Rivet-specific behaviour by try block handling & scatter.dim requirements
if os.path.basename(ao.path).startswith("_"):
continue
##
s2s.append(ao.mkScatter())
del aos #< pro-active YODA memory clean-up
#
for s2 in filter(lambda x:x.dim==2, s2s): # Filter for Scatter1D
bins = [DataBin(p.xMin, p.xMax, p.y, p.yErrAvg) for p in s2.points]
histos[s2.path] = Histo(bins, s2.path)
del s2s #< pro-active YODA memory clean-up
except Exception, e:
print "Can't load histos from file '%s': %s" % (path, e)
def ATLAS_2017_I1635274(in_file, out_file):
"""
Divided out the bin-width for jet pT distributions
add Poisson error on data
"""
data = yoda.read(in_file)
new_rivet = []
scaled_by_bin_width = ['d01', 'd03', 'd05', 'd07', 'd09']
to_be_scaled_by_bin_width = ['d02', 'd04', 'd06', 'd08', 'd10', 'd11']
for key, value in data.iteritems():
short_key = key[25:28]
# print(key, short_key)
for p in value.points:
bin_width = p.xErrs.minus + p.xErrs.plus
if abs(p.yErrs.minus) < 1E-5:
if short_key in scaled_by_bin_width:
## such distributions have already scaled by bin-width
y_err = math.sqrt(p.y*bin_width)/bin_width
else:
y_err = math.sqrt(p.y)
p.yErrs = (y_err, y_err)
else:
p.yErrs = (p.yErrs.minus, p.yErrs.plus)
if short_key in to_be_scaled_by_bin_width:
# scale by bin-width
for p in value.points:
bin_width = p.xErrs.minus + p.xErrs.plus
p.yErrs = (p.yErrs.minus/bin_width, p.yErrs.plus/bin_width)
p.y = p.y / bin_width
new_rivet.append(value)
yoda.write(new_rivet, out_file)
keywords.append({
"name": "reactions",
"values": ["BBAR0 --> D*(2010)+ LEPTON- NUBAR"]
})
keywords.append({"name": "observables", "values": ["WIDTH"]})
keywords.append({"name": "phrases", "values": ["Decay", "Charm Production"]})
reaction = r"$\bar{B}^0\to D^{*+}\ell^-\bar{\nu}_\ell$"
# metadata for whole submission
submission = [{"comment": comment}]
# alternative: single YAML file
single_yaml = [{"comment": comment}]
# read YODA file as a list of objects
objects = yoda.read("BELLE_2017_I1512299.yoda", asdict=False)
# loop over YODA objects, writing one YAML data table for each one
for iobj, object in enumerate(objects):
# define variable names depending on dimension of YODA object
if object.dim < 3:
ind_name = r"${}$".format(variables[iobj])
dep_name = r"$\Delta\Gamma/\Delta {}$".format(variables[iobj])
dep_units = r"$10^{-15}$ GeV"
dep_scale = 1e-15 # scale rates to match numbers in Table 3 of paper
description = r"The unfolded differential rate as a function of ${}$.".format(
variables[iobj])
else:
ind_name = "Bin"
dep_name = "Correlation"
def get_yoda_stats_dict(input_filename,
key_label,
data_ak4_dirname=None,
data_ak8_dirname=None,
ignore_missing=False):
"""Get summary statistics from YODA file full of histograms
Parameters
----------
input_filename : str
YODA filename
key_label : str
Label to append to column names, e.g. mean_X, rms_err_X
data_ak4_dirname : None, optional
Name of directory with AK4 data results for delta calculation
data_ak8_dirname : None, optional
Name of directory with AK8 data results for delta calculation
ignore_missing : bool, optional
If True, fill in 0 values for missing hists; otherwise raise KeyError
Returns
-------
list(dict)
List of dicts. Each dict represents one radius/region/angle combination,
with keys/values as metric names/values
Raises
------
KeyError
Description
"""
yoda_dict = yoda.read(input_filename)
hname = list(yoda_dict.keys())[0]
is_dijet = rn.DIJET_PATH in hname
regions = rn.DIJET_REGIONS if is_dijet else rn.ZPJ_REGIONS
pt_bins = rn.PT_BINS_DIJET if is_dijet else rn.PT_BINS_ZPJ
path = rn.DIJET_PATH if is_dijet else rn.ZPJ_PATH
results_dicts = []
# print("parsing", input_filename)
# print("regions:", regions)
for jet_radius in rn.JET_RADII:
algo_name = "%spuppi" % jet_radius.name.lower()
for region in regions:
for lambda_var in rn.LAMBDA_VARS:
# Get data results for delta calculation
unfolding_dict = None
data_dirname = None
if jet_radius.name.lower() == "ak4" and data_ak4_dirname:
data_dirname = data_ak4_dirname
elif jet_radius.name.lower() == "ak8" and data_ak8_dirname:
data_dirname = data_ak8_dirname
if data_dirname:
angle_output_dir = "%s/%s" % (region.name, lambda_var.hist_name)
root_filename = os.path.join(data_dirname, angle_output_dir, "unfolding_result_slim.root")
uproot_file = uproot.open(root_filename)
unfolding_dict = unpack_slim_unfolding_root_file_uproot(uproot_file, region.name, lambda_var.hist_name, pt_bins)
# Iterate over each pt bin for this radius/region/var, get metrics
for ibin, pt_bin in enumerate(pt_bins):
# print(jet_radius, lambda_var, region, ibin, pt_bin)
yoda_name = rn.get_plot_name(jet_radius, region, lambda_var, pt_bin)
yoda_name = "/%s/%s" % (path, yoda_name)
if yoda_name not in yoda_dict:
if ignore_missing:
print("Warning, missing", yoda_name, "filling with 0s")
result_dict = {
'jet_algo': algo_name,
'region': region.name,
'isgroomed': region.is_groomed,
'pt_bin': ibin,
'angle': lambda_var.hist_name,
'mean_%s' % key_label: 0,
'mean_err_%s' % key_label: 0,
'rms_%s' % key_label: 0,
'rms_err_%s' % key_label: 0,
'delta_%s' % key_label: 0,
'delta_err_%s' % key_label: 0,
}
results_dicts.append(result_dict)
else:
raise KeyError("Missing hist %s" % yoda_name)
hist = yoda_dict[yoda_name]
areas, widths, centers, errors = metrics.yoda_hist_to_values(hist)
normalize_areas(areas, errors)
areas, centers = metrics.hist_values_to_uarray(areas, centers, errors)
mean_u = metrics.calc_mean_ucert(areas, centers)
mean, mean_err = mean_u.nominal_value, mean_u.std_dev
rms_u = metrics.calc_rms_ucert(areas, centers)
rms, rms_err = rms_u.nominal_value, rms_u.std_dev
if "d11-x02-y13" in yoda_name:
print("d11-x02-y13:", mean, mean_err)
if not (0 < mean < 150):
print("WARNING bad mean:", mean, yoda_name)
delta, delta_err = 0, 0
if unfolding_dict:
unfolded_hist_bin_total_errors = unfolding_dict['unfolding_total_err_hists'][ibin]
ematrix = scale_ematrix_by_bin_widths(unfolding_dict['unfolding_total_ematrices'][ibin].values,
metrics.get_uproot_th1_bin_widths(unfolded_hist_bin_total_errors))
check_hist_for_negatives(unfolded_hist_bin_total_errors)
areas_a, widths_a, centers_a, errors_a = metrics.uproot_th1_to_arrays(unfolded_hist_bin_total_errors)
areas_b, widths_b, centers_b, errors_b = metrics.yoda_hist_to_values(hist)
if areas_a.shape != areas_b.shape:
if ibin == 0:
warnings.warn(cu.pcolors.WARNING + "%s %s %s: cannot calculate delta, areas.shape mismatch" % (algo_name, region.name, lambda_var.hist_name) + cu.pcolors.ENDC)
else:
# print(centers_a)
# print(widths_a)
# print(centers_b)
# print(widths_b)
delta = metrics.calc_delta_jax(areas_a, areas_b)
delta_err = metrics.calc_delta_correlated_error_jax(areas_a, ematrix, areas_b, errors_b)
result_dict = {
'jet_algo': algo_name,
'region': region.name,
'isgroomed': region.is_groomed,
'pt_bin': ibin,
'angle': lambda_var.hist_name,
'mean_%s' % key_label: mean,
'mean_err_%s' % key_label: mean_err,
'rms_%s' % key_label: rms,
'rms_err_%s' % key_label: rms_err,
'delta_%s' % key_label: float(delta),
'delta_err_%s' % key_label: float(delta_err),
}
results_dicts.append(result_dict)
return results_dicts
# import ROOT
# import plotting as plot
from array import array
import math
import sys
import numpy as np
import json
# import argparse
import yoda
# hname = '/HiggsTemplateCrossSectionsStage1/HTXS_stage1_pTjet30'
hname = sys.argv[2]
aos = yoda.read("Rivet.yoda", asdict=False)
# print aos
hists = [h for h in aos if h.path.startswith(hname)]
hists = hists[1:] # skip the first one
print hists
for h in hists:
print h.path, h.sumW(), math.sqrt(h.sumW2())
# help(hists[0])
nbins = hists[0].numBins
vals = np.array([h.areas() for h in hists])
print vals
with open(sys.argv[1]) as jsonfile:
pars = json.load(jsonfile)
import yoda, rivet, os
from math import sqrt
inFile = 'OUTPUT.yoda'
hists = yoda.read( inFile )
tags = sorted(hists.keys())
# probably more fancy than it needs to be ...
def getRivetRefData(anas=None):
"Find all Rivet reference data files"
refhistos = {}
for var in ("RIVET_ANALYSIS_PATH", "RIVET_DATA_PATH", "RIVET_REF_PATH", "RIVET_INFO_PATH", "RIVET_PLOT_PATH"):
if var in os.environ:
abspaths = map(os.path.abspath, os.environ[var].split(":"))
os.environ[var] = ":".join(abspaths)
rivet_data_dirs = rivet.getAnalysisRefPaths()
dirlist = [ ]
for d in rivet_data_dirs:
import glob
if anas is None:
dirlist.append(glob.glob(os.path.join(d, '*.yoda')))
else:
for a in anas:
dirlist.append(glob.glob(os.path.join(d, a+'*.yoda')))
for filelist in dirlist:
for infile in filelist:
analysisobjects = yoda.read(infile)
for path, ao in analysisobjects.iteritems():
if not any(x in path for x in ['d01', 'd02', 'd03', 'd04']): continue
def readFile(path):
inList = yoda.read(path, asdict=False)
return inList
#! /usr/bin/env python
# Load a profile histo
import yoda
aos = yoda.read("./yoda/ZHvvbb.yoda")
do_h1 = 0
do_h2 = 0
do_p1 = 1
if do_h1:
h = aos["/RivetHbbBJets/h1_PTnm_0"]
assert type(h) is yoda.Histo1D
bins = h.bins
print bins
if do_h2:
h = aos["/RivetHbbBJets/h2_PTmt_PTnm_0"]
assert type(h) is yoda.Histo2D
bins = h.bins
print bins
if do_p1:
h = aos["/RivetHbbBJets/p1_PTmt_PTnm_0"]
assert type(h) is yoda.Profile1D
bins = h.bins
print bins
#aos = yoda.read("./yoda/ZHvvbb2.yoda")
#p = aos["/RivetHbbBJets/p1_PTm_PTnm"]
#assert type(p) is yoda.Profile1D
import yoda
aosref=yoda.read("emubB_homemade.yoda")
aosHalf=yoda.read("emubB_homemade0.5.yoda")
aosDouble=yoda.read("emubB_homemade2.0.yoda")
aos = [ ao for ao in yoda.read("emubB_homemade.yoda", asdict=False) ]
aosHalf = [ao for ao in yoda.read("emubB_homemade0.5.yoda", asdict=False)]
aosDouble = [ao for ao in yoda.read("emubB_homemade2.0.yoda", asdict=False)]
result = [ ao.mkScatter() for ao in yoda.read("emubB_homemade.yoda", asdict=False)]
for i in range(0, len(aos)-3):
for p in range(0, aos[i].numBins):
Half=aosHalf[i].bins[p].sumW
Double=aosDouble[i].bins[p].sumW
errP = max(Half, Double) - aos[i].bins[p].sumW
errM = aos[i].bins[p].sumW - min(Half, Double)
result[i].points[p].y = aos[i].bins[p].sumW
result[i].points[p].yErrs = [max(errM, 0), max(errP, 0)]
yoda.write(result, "emubB_homemade_scales.yoda")
try:
bv = x.mean
except:
bv = 0.0
tp1.Fill(x.xMid, bv, x.effNumEntries)
return tp1
if __name__ =='__main__':
import os, sys, ROOT
ROOT.gROOT.SetBatch()
in_dir = sys.argv[1]
in_file = [x for x in os.listdir(in_dir) if x.endswith(".yoda")]
print in_file
for f in in_file:
tmp = yoda.read(f)
key_l = tmp.keys()
hist_l = []
for x in key_l:
tmp_h = tmp.get(x)
if tmp_h.type == "Histo1D":
tmp_h1 = Histo1D(tmp_h)
hist_l.append(tmp_h1.makeTH1F())
if tmp_h.type == "Histo2D":
tmp_h2 = Histo2D(tmp_h)
hist_l.append(tmp_h2.makeTH2D())
if tmp_h.type == "Profile1D":
tmp_p1 = Profile1D(tmp_h)
hist_l.append(tmp_p1.makeTProfile())
out_f = ROOT.TFile(f.replace(".yoda", ".root").split("/")[-1], "RECREATE")
with open(args.translate_tex) as jsonfile:
translate_tex = json.load(jsonfile)
translate_txt = {}
if args.translate_txt is not None:
with open(args.translate_txt) as jsonfile:
translate_txt = json.load(jsonfile)
bin_labels = {}
if args.bin_labels is not None:
with open(args.bin_labels) as jsonfile:
bin_labels = json.load(jsonfile)
hname = args.hist
aos = yoda.read(args.input, asdict=True)
n_pars = len(pars)
n_hists = 1 + n_pars * 2 + (n_pars * n_pars - n_pars) / 2
hists = []
for i in xrange(n_hists):
if args.nlo:
hists.append(aos['%s[rw%.4i_nlo]' % (hname, i)])
else:
hists.append(aos['%s[rw%.4i]' % (hname, i)])
# print hists
is2D = isinstance(hists[0], yoda.Histo2D)
if args.rebin is not None and not is2D:
#
# Usage:
# compute-efficiencies-pp.py <zjet-file>.yoda <dijet-file.yoda> <output.yoda>
#
# produce the efficiency measure from the two yoda files
#
# The output is under the format of another yoda file. At the moment,
# only the average weights are meaningfull.
import sys,math,re,os,yoda
if len(sys.argv) != 4:
sys.exit("Usage: compute-efficiencies-pp-new.py Zjet.yoda dijet.yoda output.yoda")
# open the files
all_zj = yoda.read(sys.argv[1])
all_jj = yoda.read(sys.argv[2])
# print what the output looks like
print '{0:21s} {1:8s} {2:8s} {3:8s} {4:8s} {5:8s} {6:8s} {7:8s}'.format("#observable","jj^rej_20","jj^rej_50","zj^rej_20","zj^rej_50","s^rej","I1/2","I1/2''")
#------------------------------------------------------------------------
# a couple of helpers
# read from a file until the line matched a pattern
def read_until_matching(fin, pattern):
line=""
while not pattern.match(line):
line=fin.readline()
if not line: sys.exit()
return line
output_object.normalize(normto)
return output_object
parser = optparse.OptionParser(usage=__doc__)
parser.add_option('-o', '--output', default='-', dest='OUTPUT_FILE')
parser.add_option('-N', '--normalize-all', action="store_true", default=False, dest='NORMALIZE_ALL')
opts, fileargs = parser.parse_args()
## Put the incoming objects into a dict from each path to a list of histos and scalings
analysisobjects_in = {}
for fa in fileargs:
filename, scale = fa, 1.0
if ":" in fa:
try:
filename, scale = fa.rsplit(":", 1)
scale = float(scale)
except:
sys.stderr.write("Error processing arg '%s' with file:scale format\n" % fa)
aos = yoda.read(filename)
for aopath, ao in aos.iteritems():
ao.setAnnotation("yodamerge_scale", scale)
analysisobjects_in.setdefault(aopath, []).append(ao)
analysisobjects_out = {}
for path, aos in analysisobjects_in.iteritems():
if(path in EFFICIENCIES): analysisobjects_out[path] = MergeEfficiency(path, aos)
else: analysisobjects_out[path] = MergeDistribution(path, aos)
yoda.writeYODA(analysisobjects_out, opts.OUTPUT_FILE)
import yoda, rivet, os
from math import sqrt
inFile = 'OUTPUT.yoda'
hists = yoda.read(inFile)
tags = sorted(hists.keys())
# probably more fancy than it needs to be ...
def getRivetRefData(anas=None):
"Find all Rivet reference data files"
refhistos = {}
for var in ("RIVET_ANALYSIS_PATH", "RIVET_DATA_PATH", "RIVET_REF_PATH",
"RIVET_INFO_PATH", "RIVET_PLOT_PATH"):
if var in os.environ:
abspaths = map(os.path.abspath, os.environ[var].split(":"))
os.environ[var] = ":".join(abspaths)
rivet_data_dirs = rivet.getAnalysisRefPaths()
dirlist = []
for d in rivet_data_dirs:
import glob
if anas is None:
dirlist.append(glob.glob(os.path.join(d, '*.yoda')))
else:
for a in anas:
dirlist.append(glob.glob(os.path.join(d, a + '*.yoda')))
for filelist in dirlist:
for infile in filelist:
analysisobjects = yoda.read(infile)
for path, ao in analysisobjects.iteritems():
#
# produce-Rdependence-avg.py generator level output_file
#
import sys,re,yoda,os,math
if len(sys.argv) != 4:
sys.exit("Usage: produce-Rdependence-avg.py generator level output_file")
# open the files
gen = sys.argv[1]
level = sys.argv[2]
observables=["GA_10_05", "GA_10_10", "GA_10_20"]
flavours=["uu", "gg"]
Rvalues=[2,4, 6, 8,10]
scatters=[]
for flavour in flavours:
for observable in observables:
separation_scatter = yoda.Scatter2D(title=observable, path="/Rdependence/"+flavour+"_"+observable)
hnamebase="/MC_LHQG_EE/"+observable+"_R"
dict_all=yoda.read(gen+"/"+level+"/"+flavour+"-200.yoda", True)
for Rval in Rvalues:
separation_scatter.addPoint(0.1*Rval, dict_all[hnamebase+str(Rval)].xMean(), xerrs=0.1)
# add that for stddev as errorbars: , yerrs=dict_all[hnamebase+str(Rval)].xStdDev())
scatters.append(separation_scatter)
yoda.write(scatters,sys.argv[3])
import yoda
import math
aos = [ ao for ao in yoda.read("WWbBPOWHEG.yoda", asdict=False) ]
result = [ ao.mkScatter() for ao in yoda.read("WWbBPOWHEG.yoda", asdict=False)]
for i in range(0, len(aos)-3):
for p in range(0, aos[i].numBins):
result[i].points[p].y = aos[i].bins[p].sumW
result[i].points[p].yErrs = [math.sqrt(aos[i].bins[p].sumW2), math.sqrt(aos[i].bins[p].sumW2)]
yoda.write(result, "WWbBPOWHEGScatter.yoda")
def toList(h):
return map(yoda.HistoBin1D.area, h.bins())
def readHists(d, names):
return concat([toList(d[name]) for name in names])
def readHistsWithVars(d, names, varnames):
return \
( readHists(d, names)
, { varname : readHists(d, [name + "[%s]" % varname for name in names]) for varname in varnames }
)
aos = yoda.read(infile)
histvariations = readHistsWithVars(aos, histnames, varnames)
def setMin(c):
def f(h):
return lmap(lambda x: max(c, x), h)
return f
def setKey(k, v, d):
d[k] = v
return d
import yoda
import math
fi=yoda.read("Rivet.yoda", asdict=False)
aos = []
for h in fi:
if h.annotation("Type") == "Histo1D":
s=yoda.Scatter2D()
for a in h.annotations:
s.setAnnotation(a, h.annotation(a))
s.setAnnotation("Type", "Scatter2D")
for b in h.bins:
x = b.xMid
ex = x - b.xMin
y = b.height
ey = b.heightErr
s.addPoint(x,y,ex,ey)
aos.append(s)
else: aos.append(h)
print aos
yoda.write(aos, "Rivet_scatter.yoda")
#
import sys,re,yoda,os,math
if len(sys.argv) != 4:
sys.exit("Usage: produce-Qdependence-avg.py generator level output_file")
# open the files
gen = sys.argv[1]
level = sys.argv[2]
observables=["GA_10_05_R6", "GA_10_10_R6", "GA_10_20_R6"]
flavours=["uu", "gg"]
scatters=[]
for flavour in flavours:
for observable in observables:
separation_scatter = yoda.Scatter2D(title=observable, path="/Qdependence/"+flavour+"_"+observable)
for Q in [50,100,200,400,800]:
hname="/MC_LHQG_EE/"+observable
dict_all=yoda.read(gen+"/"+level+"/"+flavour+"-"+str(Q)+".yoda", True)
Qmin=Q/math.sqrt(2.0)
Qmax=Q*math.sqrt(2.0)
separation_scatter.addPoint(Q, dict_all[hname].xMean(), xerrs=[Q-Qmin,Qmax-Q])
# add that for stddev as errorbars: , yerrs=dict_all[hname].xStdDev())
scatters.append(separation_scatter)
yoda.write(scatters,sys.argv[3])
#"EWWZ_ATLAS_Sherpa_222_NNPDF30NNLO.v1.yoda",
"EWWZ_ATLAS_Sherpa.yoda",
"EWWZ_ATLAS_MGPy8_full.yoda",
#"WZJJ_EW_MG_LHConfig_MaxPtJScale.yoda"
#"WZJJ_EW_CMSMGSample_LooseFiducial.yoda",
#"WZJJ_EW_MG_LHConfig_LooseFiducial.yoda",
#"WZJJ_EW_LHConfig_powheg-pythia8_LooseFiducial.yoda",
]
#"WmZTo1E1Nu2Mu_FixedScaleMW_LHConfig_VBFNLO-Pythia8.yoda",
binning = {
"Mass3l": [4],
"dEtajj": [2],
"mjj": [5],
"zep3l": [2],
"zepj3": [2],
"dRjj": [2],
}
for filename in filenames:
fullname = "/".join([data_path, filename])
hists = yoda.read(fullname)
for name, hist in hists.iteritems():
name = name.split("/")[-1]
if not type(hist) is yoda.Histo1D:
continue
key = name.split("_")[-1]
if key not in binning.keys():
continue
hist.rebin(*binning[key])
yoda.write(hists, fullname.replace(".yoda", "_REBIN.yoda"))
