def load_all_yaml_files(dataset_, path_, **kwargs):
do_recurse = kwargs.get("recurse", True)
path_ = hlp.remove_subleading(path_, "/")
if hlp.is_directory(path_):
yaml_files = hlp.keep_only_yaml_files(path_, recurse=do_recurse)
if len(yaml_files) == 0:
msg.error(
"HEP_data_helpers.load_all_yaml_files",
"Directory {0} has no yaml files... returning with nothing done."
.format(path_),
verbose_level=-1)
return
for f in yaml_files:
load_all_yaml_files(dataset_, f, **kwargs)
return
if not hlp.is_yaml_file(path_):
msg.error(
"HEP_data_helpers.load_all_yaml_files",
"Path {0} is not a directory or yaml file... returning with nothing done."
.format(path_),
verbose_level=-1)
return
if hlp.is_submission_file(path_):
load_submission_file(dataset_, path_, **kwargs)
return
load_yaml_file(dataset_, path_, **kwargs)
def load_yaml_file(dataset_, path_, **kwargs):
msg.info("HEP_data_helpers.load_yaml_file",
"Opening yaml file {0}".format(path_),
verbose_level=0)
data = open_yaml_file(path_)
if len(data) != 1:
msg.error(
"HEP_data_helpers.load_yaml_file",
"{0} contains {1} tables where I was expecting 1... is the file format correct?"
.format(path_, len(data)))
return
dep_vars = data[0].get("dependent_variables", None)
if dep_vars is None:
msg.fatal(
"HEP_data_helpers.load_yaml_file",
"{0} contains no dependent_variables as required".format(path_))
return
indep_vars = data[0].get("independent_variables", None)
if indep_vars is None:
msg.fatal(
"HEP_data_helpers.load_yaml_file",
"{0} contains no independent_variables as required".format(path_))
return
load_distributions_from_yaml(dataset_, dep_vars, indep_vars, path_,
**kwargs)
def parse_inputs ( argv_ ) :
# Get arguments
try :
opts, rest = getopt.getopt(argv_,"hrv:",["help","recursive","verbosity=","yaml=","compare=",""])
except getopt.GetoptError as err :
msg.error("validate_yaml_files.py","The following error was thrown whilst parsing command-line arguments")
print(">>>>>>>>\n",err,"\n<<<<<<<<")
msg.error("validate_yaml_files.py","Falling back to to --help...")
print_help()
msg.fatal("validate_yaml_files.py","Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
for opt, arg in opts:
if opt in ['-h',"--help"] :
print_help()
sys.exit(0)
if opt in ['-r',"--recursive",] :
msg.info("validate_yaml_files.py","Config: using recursion if needed",verbose_level=0)
do_recurse = True
if opt in ['-v',"--verbosity"] :
msg.info("validate_yaml_files.py","Config: setting verbosity to {0}".format(arg),verbose_level=0)
try : msg.VERBOSE_LEVEL = int(arg)
except : msg.fatal("validate_yaml_files.py","Could not cast verbosity level {0} to integer".format(arg))
yaml_files = hlp.keep_only_yaml_files(get_argument_list(argv_,"--yaml"),recurse=do_recurse)
root_files = hlp.keep_only_root_files(get_argument_list(argv_,"--compare"),recurse=do_recurse)
if len(yaml_files) == 0 :
msg.fatal("validate_yaml_files.py","Please provide at least one yaml file using the --yaml option")
if len(root_files) == 0 :
msg.fatal("validate_yaml_files.py","Please provide at least one root file using the --compare option")
# Return
return yaml_files, root_files
def set_save_file ( fname_ ) :
global document
if type(fname_) is str :
if type(document) is PdfPages : document.close()
if fname_[-4:] != ".pdf" : fname_ = fname_ + ".pdf"
msg.info("HEP_data_utils.plotting.set_save_file","Opening pdf file {0}".format(fname_),verbose_level=0)
document = PdfPages(fname_)
else : msg.error("HEP_data_utils.plotting.set_save_file","Filename must be a str")
def asymerrors_dn(self, key_=None):
if key_ is None: return self._asymerrors_dn
if key_ not in self._asymerrors_dn:
msg.error(
"DependentVariable.asymerrors_dn",
"Key {0} not found in self._asymerrors_dn... returning None.".
format(key_),
verbose_level=0)
return None
return self._asymerrors_dn[key_]
def symerrors(self, key_=None):
if key_ is None: return self._symerrors
if key_ not in self._symerrors:
msg.error(
"DependentVariable.symerrors",
"Key {0} not found in self._symerrors... returning None.".
format(key_),
verbose_level=0)
return None
return self._symerrors[key_]
def plot_1D_distribution ( table_ , **kwargs ) :
if table_.n_indep_vars() != 1 :
msg.error("plotting.plot_1D_distribution","Table has {0} independent variables where 1 was expected".format(table_.n_indep_vars()))
return
fig = plt.figure(figsize=(10,5))
ax = fig.add_subplot(111)
legend_char_width = 53
specific_errors = []
for requested_error in kwargs.get("errors",[]) :
if requested_error not in table_._dep_var._symerrors and requested_error not in table_._dep_var._asymerrors_up :
msg.warning("plotting.plot_1D_distribution","Specified error {0} not found in dependent variable {1}. Ignoring.".format(requested_error,table_._dep_var._name))
continue
specific_errors.append(requested_error)
if len(specific_errors) > 0 :
x, y, [ex_lo,ex_hi], [ey_lo,ey_hi], labels, keys = get_1D_distribution(table_,specific_errors)
ey_lo_sys, ey_hi_sys, ey_lo_stat, ey_hi_stat = [], [], [], []
str_tot_legend = kwargs.get("label","distribution") + " ( " + " + ".join(keys) + " )"
str_tot_legend = "\n".join([str_tot_legend[legend_char_width*i:min(len(str_tot_legend),legend_char_width*(i+1))] for i in range(int(len(str_tot_legend)/legend_char_width)+1)])
else :
x, y, [ex_lo,ex_hi], [ey_lo,ey_hi], labels, keys = get_1D_distribution(table_)
x, y, [ex_lo,ex_hi], [ey_lo_sys,ey_hi_sys], labels, sys_keys = get_1D_distribution(table_,"sys")
x, y, [ex_lo,ex_hi], [ey_lo_stat,ey_hi_stat], labels, stat_keys = get_1D_distribution(table_,"stat")
str_tot_legend = kwargs.get("label","distribution") + " ( " + " + ".join(keys) + " )"
str_tot_legend = "\n".join([str_tot_legend[legend_char_width*i:min(len(str_tot_legend),legend_char_width*(i+1))] for i in range(int(len(str_tot_legend)/legend_char_width)+1)])
str_sys_legend = kwargs.get("label","distribution") + " ( " + " + ".join(sys_keys) + " )"
str_sys_legend = "\n".join([str_sys_legend[legend_char_width*i:min(len(str_sys_legend),legend_char_width*(i+1))] for i in range(int(len(str_sys_legend)/legend_char_width)+1)])
if sum([np.fabs(el) for el in ey_hi_sys+ey_lo_sys]) > 0 :
ax.errorbar(x, y, yerr=[ey_lo_sys,ey_hi_sys], c='royalblue', linewidth=18, linestyle='None', marker='None', alpha=0.4, label=str_sys_legend)
if sum([np.fabs(el) for el in ey_hi_stat+ey_lo_stat]) > 0 :
ax.errorbar(x, y, yerr=[ey_lo_stat,ey_hi_stat], c='indianred', linewidth=6, linestyle='None', marker='None', alpha=0.6, label=kwargs.get("label","distribution") + " ( stat )")
ax.errorbar(x, y, yerr=[ey_lo,ey_hi], xerr=[ex_lo,ex_hi], c='k', linewidth=2, linestyle='None', marker='+', alpha=1, label=str_tot_legend)
if labels :
ax.set_xticks(x)
ax.set_xticklabels(table_._indep_vars[0]._bin_labels,rotation=90)
plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4)
if "legend_loc" in kwargs : ax.legend(loc=kwargs.get("legend_loc","best"))
else : ax.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
try : plt.xlabel(kwargs.get("xlabel",table_._indep_vars[0].name().replace("\\\\","\\").replace(r"\text{",r"{\rm ")))
except : plt.xlabel("<error reading xlabel")
try : plt.ylabel(kwargs.get("ylabel",table_._dep_var.name().replace("\\\\","\\").replace(r"\text{",r"{\rm ")))
except : plt.ylabel("<error reading ylabel")
plt.title(kwargs.get("title",""))
xlim = kwargs.get("xlim",[x[0]-np.fabs(ex_lo[0]),x[-1]+np.fabs(ex_hi[-1])])
ylim = kwargs.get("ylim",None)
ax.axis(xmin=xlim[0],xmax=xlim[1])
if ylim : ax.axis(ymin=ylim[0],ymax=ylim[1])
if kwargs.get("logy",False) is True : plt.yscale("log")
if kwargs.get("logx",False) is True : plt.xscale("log")
plt.grid()
if kwargs.get("show",False) :
plt.show()
if kwargs.get("save",False) :
fig.savefig ( document , format='pdf' )
plt.close(fig)
def load_root_files_from_list ( dataset_ , dir_ , **kwargs ) :
if hlp.is_directory(dir_) :
for filename in [ dir_+"/"+f for f in os.listdir(dir_) if is_root_file(f) ] :
msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening root file {0}".format(filename),verbose_level=0)
load_all_root_files(dataset_,filename,**kwargs)
elif type(dir_) == list :
for filename in dir_ :
if type(filename) != str : continue
msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening yaml file {0}".format(filename),verbose_level=0)
load_all_root_files(dataset_,filename,**kwargs)
else :
msg.error("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Input {0} is neither a directory nor a list... returning with nothing done".format(dir_),verbose_level=-1)
def remove_None_entries_from_1D_table(table_):
# check dimensions and number of Nones
if table_.n_indep_vars() != 1: return
dep_var = table_._dep_var
indep_var = table_._indep_vars[0]
old_values, old_labels, old_centers, old_bw_lo, old_bw_hi = dep_var._values, indep_var._bin_labels, indep_var._bin_centers, indep_var._bin_widths_lo, indep_var._bin_widths_hi
num_Nones = len([x for x in old_values if x is None])
if num_Nones == 0: return
# make sure bin labels are sensibly configured for a discontinuous distribution
if len(old_values) != len(old_labels):
msg.error(
"HEP_data_utils.HEP_data_helpers.remove_None_entries_from_1D_table",
"Cannot interpret binning when len(values) = {0} but len(labels) = {1}"
.format(len(old_values), len(old_labels)),
verbose_level=1)
return
for i in range(len(old_labels)):
label = old_labels[i]
if len(label) > 0: continue
old_labels[i] = "[{0},{1}]".format(old_centers[i] - old_bw_lo[i],
old_centers[i] + old_bw_hi[i])
# set new values
None_indices = []
new_labels = []
new_bw_lo, new_bw_hi = np.full(shape=(len(old_bw_lo) - num_Nones),
fill_value=0.5), np.full(
shape=(len(old_bw_hi) - num_Nones),
fill_value=0.5)
new_centers = np.zeros(shape=(len(old_centers) - num_Nones))
for i in range(len(new_centers)):
new_centers[i] = i
new_bw_lo[i], new_bw_hi[i] = 0.5, 0.5
indep_var._bin_centers, indep_var._bin_widths_lo, indep_var._bin_widths_hi = new_centers, new_bw_lo, new_bw_hi
for i in range(len(old_values)):
if old_values[i] == None:
None_indices.append(i)
continue
new_labels.append(old_labels[i])
dep_var._values = np.delete(old_values, None_indices)
indep_var._bin_labels = new_labels
# set new errors
for key in dep_var._symerrors:
dep_var._symerrors[key] = np.delete(dep_var._symerrors[key],
None_indices)
for key in dep_var._asymerrors_up:
dep_var._asymerrors_up[key] = np.delete(dep_var._asymerrors_up[key],
None_indices)
for key in dep_var._asymerrors_dn:
dep_var._asymerrors_dn[key] = np.delete(dep_var._asymerrors_dn[key],
None_indices)
def rename_key(self, old_key_, new_key_):
something_done = False
old_key = r"{0}".format(old_key_)
new_key = r"{0}".format(new_key_)
for key in self._inclusive_distributions:
if old_key != key: continue
self._inclusive_distributions[
new_key_] = self._inclusive_distributions.pop(old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming inclusive distribution key {1} to {2}".
format(self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
for key in self._1D_distributions:
if old_key != key: continue
self._1D_distributions[new_key_] = self._1D_distributions.pop(
old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming 1D distribution key {1} to {2}".format(
self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
for key in self._2D_distributions:
if old_key != key: continue
self._2D_distributions[new_key_] = self._2D_distributions.pop(
old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming 2D distribution key {1} to {2}".format(
self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
for key in self._ND_distributions:
if old_key != key: continue
self._ND_distributions[new_key_] = self._ND_distributions.pop(
old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming ND distribution key {1} to {2}".format(
self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
if not something_done:
msg.error(
"DistributionContainer.rename_key",
"Store \"{0}\" with nothing done for old_key_={1}, new_key_={2}"
.format(self._name, old_key_, new_key_),
verbose_level=1)
def load_distribution_from_ROOT_Table(dataset_, key_, table_, **kwargs):
# Create the table object
table = HEPDataTable()
table._submission_file_meta = kwargs.get("submission_file_metadata", None)
table._submission_file_table = kwargs.get("submission_file_table", None)
# Set the variables
table._dep_var = ROOT_observable_to_DependentVariable(table_.dep_var)
for indep_var in table_.indep_vars:
table._indep_vars.append(ROOT_axis_to_IndependentVariable(indep_var))
# Figure out what key to give it
if key_ in dataset_:
key_ = key_ + "-;1"
while key_ in self:
key_ = key_[:-1] + str(1 + int(key_[-1:]))
# Validate our table
is_valid, validity_message = table.is_valid()
if not is_valid:
msg.error(
"ROOT_helpers.load_distribution_from_ROOT_Table",
"Error occured when loading table {0}... returning with nothing done."
.format(key_))
msg.error("ROOT_helpers.load_distribution_from_ROOT_Table",
">>>>>>>>>>>>>>")
msg.error("ROOT_helpers.load_distribution_from_ROOT_Table",
validity_message)
msg.error("ROOT_helpers.load_distribution_from_ROOT_Table",
"<<<<<<<<<<<<<<")
return
# Add to dataset
n_dim = table.n_indep_vars()
if n_dim == 0: dataset_._inclusive_distributions[key_] = table
elif n_dim == 1: dataset_._1D_distributions[key_] = table
elif n_dim == 2: dataset_._2D_distributions[key_] = table
else: dataset_._ND_distributions[key_] = table
def plot_2D_distribution_on_current_axes(table_, **kwargs):
if table_.n_indep_vars() != 2:
msg.error(
"HEP_data_utils.plotting.plot_2D_distribution_on_current_canvas",
"Table has {0} independent variables where 1 was expected".format(
table_.name(), table_.n_indep_vars()))
return
dep_var, indep_var = table_._dep_var, table_._indep_vars
values = dep_var._values
ax = plt.gca()
try:
plt.xlabel(
kwargs.get(
"x_label",
indep_var[0].name().replace("\\\\",
"\\").replace(r"\text{",
r"{\rm ")))
except Exception:
pass
try:
plt.ylabel(
kwargs.get(
"y_label",
indep_var[1].name().replace("\\\\",
"\\").replace(r"\text{",
r"{\rm ")))
except Exception:
pass
max_val = max([np.fabs(val) for val in values.flatten()])
vmin, vmax = -1 * max_val, max_val
if "vlim" in kwargs:
vmin = kwargs["vlim"][0]
vmax = kwargs["vlim"][1]
labels_x, labels_y, values = get_2D_distribution(table_)
ax.imshow(values.transpose(), cmap="bwr", vmin=vmin, vmax=vmax)
precision = kwargs.get("flt_precision", 2)
for i in range(len(labels_x)):
for j in range(len(labels_y)):
ax.text(i,
j,
"{0:.{1}f}".format(values[i, j], precision),
ha="center",
va="center",
color="k",
fontsize=kwargs.get("fontsize", "xx-small"))
ax.set_xticks(np.arange(len(labels_x)))
ax.set_xticklabels(labels_x, rotation=90)
ax.set_yticks(np.arange(len(labels_y)))
ax.set_yticklabels(labels_y)
def plot_ratio ( table_num_ , table_den_ , **kwargs ) :
specific_errors_num = [ err for err in kwargs.get("errors",[]) if err in { **table_num_._dep_var._symerrors, **table_num_._dep_var._asymerrors_up } ]
specific_errors_den = [ err for err in kwargs.get("errors",[]) if err in { **table_den_._dep_var._symerrors, **table_den_._dep_var._asymerrors_up } ]
x_n, y_n, [ex_lo_n,ex_hi_n], [ey_lo_n,ey_hi_n], labels, keys_num = get_1D_distribution(table_num_,specific_errors_num)
x_d, y_d, [ex_lo_d,ex_hi_d], [ey_lo_d,ey_hi_d], labels, keys_den = get_1D_distribution(table_den_,specific_errors_den)
ex_lo_d, ex_hi_d = np.zeros(shape=(len(x_d))), np.zeros(shape=(len(x_d)))
for i in range(len(x_n)) :
if x_n[i] == x_d[i] : continue
msg.error("HEP_data_helpers.plot_ratio","Arguments do not have the same binning")
raise ValueError("Ratio of distributions with bin centres at {0} and {1}",x_n.all(),x_d.all())
fig = plt.figure(figsize=(10,10))
ax1 = fig.add_subplot(211)
legend_char_width = 53
str_num_legend = kwargs.get("numerator_label","numerator") + " ( " + " + ".join(keys_num) + " )"
str_num_legend = "\n".join([str_num_legend[legend_char_width*i:min(len(str_num_legend),legend_char_width*(i+1))] for i in range(int(len(str_num_legend)/legend_char_width)+1)])
str_den_legend = kwargs.get("denominator_label","denominator") + " ( " + " + ".join(keys_den) + " )"
str_den_legend = "\n".join([str_den_legend[legend_char_width*i:min(len(str_den_legend),legend_char_width*(i+1))] for i in range(int(len(str_den_legend)/legend_char_width)+1)])
ax1.errorbar(x_d, y_d, yerr=[ey_lo_d,ey_hi_d], xerr=[ex_lo_d,ex_hi_d], c='r', linewidth=7, linestyle='None', marker='+', alpha=0.5, label=str_den_legend)
ax1.errorbar(x_n, y_n, yerr=[ey_lo_n,ey_hi_n], xerr=[ex_lo_n,ex_hi_n], c='k', linestyle='None', alpha=0.5, label=str_num_legend)
if "legend_loc" in kwargs : ax1.legend(loc=kwargs.get("legend_loc","best"))
else : ax1.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4)
try : plt.ylabel(kwargs.get("ylabel",table_num_._dep_var.name().replace("\\\\","\\").replace(r"\text{",r"{\rm ")))
except : plt.ylabel("<error reading ylabel")
plt.title(kwargs.get("title",""))
xlim = kwargs.get("xlim",[x_d[0]-np.fabs(ex_lo_n[0]),x_d[-1]+np.fabs(ex_hi_n[-1])])
ylim = kwargs.get("ylim",None)
ax1.axis(xmin=xlim[0],xmax=xlim[1])
if ylim : ax1.axis(ymin=ylim[0],ymax=ylim[1])
if kwargs.get("logy",False) is True : plt.yscale("log")
if kwargs.get("logx",False) is True : plt.xscale("log")
plt.grid()
ax2 = fig.add_subplot(212)
ax2.errorbar(x_d, y_d/y_d, yerr=[ey_lo_d/y_d,ey_hi_d/y_d], xerr=[ex_lo_d,ex_hi_d], c='r', linewidth=7, linestyle='None', marker='+', alpha=0.5)
ax2.errorbar(x_n, y_n/y_d, yerr=[ey_lo_n/y_d,ey_hi_n/y_d], xerr=[ex_lo_n,ex_hi_n], c='k', linestyle='None', alpha=0.5)
box = ax2.get_position()
ax2.set_position([box.x0, box.y0, box.width * 0.4, box.height])
ax2.axis(xmin=xlim[0],xmax=xlim[1])
plt.ylabel("Ratio")
try : plt.xlabel(kwargs.get("xlabel",table_den_._indep_vars[0].name().replace("\\\\","\\").replace(r"\text{",r"{\rm ")))
except : plt.xlabel("<error reading xlabel")
plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4)
plt.grid()
if kwargs.get("show",False) :
plt.show()
if kwargs.get("save",False) :
fig.savefig ( document , format='pdf' )
plt.close(fig)
def load_submission_file(dataset_, path_, **kwargs):
data = open_yaml_file(path_)
# First get the dataset metadata
sub_file_meta = SubmissionFileMeta()
indices_of_metadata = []
for idx in range(len(data)):
dataset_properties = data[idx]
if "data_file" in dataset_properties: continue
indices_of_metadata.append(idx)
additional_resources = dataset_properties.get("additional_resources",
[])
for idx in range(len(additional_resources)):
sub_file_meta._additional_resources.append(
additional_resources[idx])
if "comment" in dataset_properties:
sub_file_meta._comment = dataset_properties["comment"]
if "hepdata_doi" in dataset_properties:
sub_file_meta._hepdata_doi = dataset_properties["hepdata_doi"]
# Now load tables from all remaining entries
for idx in range(len(data)):
if idx in indices_of_metadata: continue
table = data[idx]
data_file = hlp.get_directory(path_) + "/" + table["data_file"]
if not os.path.isfile(data_file):
msg.error(
"HEP_data_helpers.load_submission_file",
"Submission file asks for a yaml file called {0} but none exists"
.format(data_file))
return
sub_file_table = SubmissionFileTable()
sub_file_table._data_file = data_file
sub_file_table._name = table.get("name", "")
sub_file_table._table_doi = table.get("table_doi", "")
sub_file_table._location = table.get("location", "")
sub_file_table._description = table.get("description", "")
sub_file_table._keywords = [(d.get("name", ""), d.get("values", ""))
for d in table.get("keywords", [])]
load_yaml_file(dataset_,
data_file,
submission_file_metadata=sub_file_meta,
submission_file_table=sub_file_table,
**kwargs)
def load_keys(self, filename_):
config = configparser.ConfigParser()
config.optionxform = str
try:
config.read(filename_)
except Exception as exc:
msg.check_verbosity_and_print(exc, verbose_level=-1)
msg.error(
"HEP_data_utils.data_structures.DistributionContainer",
"An exception occured when parsing the config file... Continuing with nothing done"
)
return
if "KEYS" not in config.sections():
msg.error(
"HEP_data_utils.data_structures.DistributionContainer",
"No section titled \"KEYS\" in file {0}".format(filename_))
return
keys = config["KEYS"]
for old_key in keys:
self.rename_key(old_key, keys[old_key])
self.print_keys()
def get_error_from_yaml_map(dep_var_, error_, pt_idx_, err_idx_=0):
key = error_.get("label", "err{0}".format(err_idx_))
if "symerror" in error_:
if key not in dep_var_._symerrors:
msg.info("HEP_data_helpers.get_error_from_yaml_map",
"Creating symmetric error {0} with length {1}".format(
key, len(dep_var_)),
verbose_level=2)
dep_var_._symerrors[key] = np.zeros(shape=(len(dep_var_)))
dep_var_._symerrors[key][pt_idx_] = float(error_["symerror"])
elif "asymerror" in error_:
err_asymm = error_["asymerror"]
if key not in dep_var_._asymerrors_up:
msg.info("HEP_data_helpers.get_error_from_yaml_map",
"Creating asymmetric error {0} with length {1}".format(
key, len(dep_var_)),
verbose_level=2)
dep_var_._asymerrors_up[key] = np.zeros(shape=(len(dep_var_)))
dep_var_._asymerrors_dn[key] = np.zeros(shape=(len(dep_var_)))
if "plus" not in err_asymm:
msg.error("HEP_data_helpers.get_error_from_yaml_map",
"No entry named \"plus\" for error \"asymerror\"")
else:
dep_var_._asymerrors_up[key][pt_idx_] = float(err_asymm["plus"])
if "minus" not in err_asymm:
msg.error("HEP_data_helpers.get_error_from_yaml_map",
"No entry named \"minus\" for error \"asymerror\"")
else:
dep_var_._asymerrors_dn[key][pt_idx_] = float(err_asymm["minus"])
else:
print(yaml.safe_dump(error_))
msg.error("HEP_data_helpers.get_error_from_yaml_map",
"map does not have an entry called symerror or asymerror")
return key
def plot(self, key_, **kwargs):
for key, dist in self._inclusive_distributions.items():
if key != key_: continue
try:
plotter.plot_inclusive_distribution(dist, **kwargs)
except Exception as e:
print(e)
msg.error(
"HEP_data_utils.data_structures.DistributionContainer.plot",
"Error when plotting inclusive distribution with key {0}... skipping"
.format(key))
for key, dist in self._1D_distributions.items():
if key != key_: continue
try:
plotter.plot_1D_distribution(dist, **kwargs)
except Exception as e:
print(e)
msg.error(
"HEP_data_utils.data_structures.DistributionContainer.plot",
"Error when plotting 1D distribution with key {0}... skipping"
.format(key))
for key, dist in self._2D_distributions.items():
if key != key_: continue
try:
plotter.plot_2D_distribution(dist, **kwargs)
except Exception as e:
print(e)
msg.error(
"HEP_data_utils.data_structures.DistributionContainer.plot",
"Error when plotting 2D distribution with key {0}... skipping"
.format(key))
def load_distribution_from_yaml(dataset_, dep_var_, indep_vars_, path_,
**kwargs):
# Create the table object
table = HEPDataTable()
table._submission_file_meta = kwargs.get("submission_file_metadata", None)
table._submission_file_table = kwargs.get("submission_file_table", None)
# Set the dependent variable
dep_var = get_dependent_variable_from_yaml_map(dep_var_, path_)
table._dep_var = dep_var
# Set the independent variables
for indep_var_map in indep_vars_:
indep_var = IndependentVariable()
indep_header = indep_var_map.get("header", None)
if indep_header:
indep_var._name = indep_header.get("name", "")
indep_var._units = indep_header.get("units", "")
indep_var._bin_centers, indep_var._bin_widths_lo, indep_var._bin_widths_hi, indep_var._bin_labels = get_bins_from_dict(
indep_var_map, len(dep_var))
table._indep_vars.append(indep_var)
# If our table has 'None' entries, we want to remove them
remove_None_entries_from_1D_table(table)
# Figure out what key to give it
key = dataset_.generate_key(table)
# Validate our table
is_valid, validity_message = table.is_valid()
if not is_valid:
msg.error(
"HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml",
"Error occured when loading table {0} from file {1}... returning with nothing done."
.format(key, path_))
msg.error(
"HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml",
">>>>>>>>>>>>>>")
msg.error(
"HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml",
validity_message)
msg.error(
"HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml",
"<<<<<<<<<<<<<<")
return
# Reformat 2D distribution bins into a matrix
n_dim = len(table._indep_vars)
if n_dim == 2 and dataset_._make_matrix_if_possible:
reformat_2D_bins_as_matrix(table)
# Add to dataset
if n_dim == 0: dataset_._inclusive_distributions[key] = table
elif n_dim == 1: dataset_._1D_distributions[key] = table
elif n_dim == 2: dataset_._2D_distributions[key] = table
else: dataset_._ND_distributions[key] = table
def plot_2D_distribution(table_, **kwargs):
if table_.n_indep_vars() != 2:
msg.error(
"HEP_data_utils.plotting.plot_2D_distribution",
"Table has {0} independent variables where 1 was expected".format(
table_.name(), table_.n_indep_vars()))
return
dep_var, indep_var = table_._dep_var, table_._indep_vars
values = dep_var._values
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111)
plot_2D_distribution_on_current_axes(table_, **kwargs)
try:
plt.title(table_._dep_var.name().replace("\\\\", "\\").replace(
r"\text{", r"{\rm "))
except Exception:
pass
if kwargs.get("show", False):
plt.show()
if kwargs.get("save", False):
fig.savefig(document, format='pdf')
plt.close(fig)
def plot_2D_distribution ( table_ , **kwargs ) :
if table_.n_indep_vars() != 2 :
msg.error("plotting.plot_2D_distribution","Table has {0} independent variables where 1 was expected".format(table_.name(),table_.n_indep_vars()))
return
dep_var = table_._dep_var
indep_var = table_._indep_vars
values = dep_var._values
fig = plt.figure(figsize=(7,7))
ax = fig.add_subplot(111)
#x_label = str([ "{0} [{1}:{2}]".format(var,table_._local_key_indices[var][0],table_._local_key_indices[var][1]) for var in table_._local_keys ])
try : x_label = kwargs.get("x_label",indep_var[0].name().replace("\\\\","\\").replace(r"\text{",r"{\rm "))
except : plt.xlabel("<error reading xlabel")
try : y_label = kwargs.get("y_label",indep_var[1].name().replace("\\\\","\\").replace(r"\text{",r"{\rm "))
except : plt.ylabel("<error reading ylabel")
max_val = max([np.fabs(val) for val in values.flatten()])
vmin = -1*max_val
vmax = max_val
if "vlim" in kwargs :
vmin = kwargs["vlim"][0]
vmax = kwargs["vlim"][1]
labels_x, labels_y, values = get_2D_distribution(table_)
ax.imshow(values,cmap="bwr",vmin=vmin,vmax=vmax)
plt.xlabel(kwargs.get("xlabel",x_label))
plt.ylabel(kwargs.get("ylabel",y_label))
precision = kwargs.get("flt_precision",2)
for i in range(len(labels_x)) :
for j in range(len(labels_y)) :
ax.text(j, i, "{0:.{1}f}".format(values[i, j],precision), ha="center", va="center", color="k",fontsize="xx-small")
ax.set_xticks(np.arange(len(labels_x)))
ax.set_xticklabels(labels_x,rotation=90)
ax.set_yticks(np.arange(len(labels_y)))
ax.set_yticklabels(labels_y)
plt.title(kwargs.get("title",dep_var.name().replace("\\\\","\\").replace(r"\text{",r"{\rm ")))
if kwargs.get("show",False) :
plt.show()
if kwargs.get("save",False) :
fig.savefig ( document , format='pdf' )
plt.close(fig)
def plot_and_print(num_dist, den_dist, **kwargs):
if not HD.has_matching_bins(num_dist, den_dist): return
n_dim = num_dist.n_indep_vars()
print(
"====================================================================================="
)
print("=== PLOTTING THE FOLLOWING RATIO ( 1 divided by 2 )")
print("=== 1. ", num_dist._dep_var._name)
print("=== 2. ", den_dist._dep_var._name)
if n_dim == 1:
print("=== as a function of: ", num_dist._indep_vars[0]._name)
if n_dim == 2:
print("=== as a function of: ", num_dist._indep_vars[0]._name)
print("=== ", num_dist._indep_vars[1]._name)
try:
chi2, bin_ratios = plot_ratio(num_dist, den_dist, **kwargs)
print("=== chi2 = {:.4f}".format(chi2))
print("=== bin ratios = ", ["{:.4f}".format(x) for x in bin_ratios])
except Exception as e:
print(e)
msg.error("plot_ratios", "Error when plotting ratio... skipping")
print(
"====================================================================================="
)
def get_dependent_variable_from_yaml_map(dep_var_, path_="unknown"):
dep_var = DependentVariable()
dep_header = dep_var_.get("header", None)
if dep_header:
dep_var._name = dep_header.get("name", "")
dep_var._units = dep_header.get("units", "")
dep_qualifiers = dep_var_.get("qualifiers", None)
if dep_qualifiers:
if type(dep_qualifiers) is not list:
msg.error(
"HEP_data_helpers.get_dependent_variable_from_yaml_map",
"Dependent variable {0} in file {1} has qualifiers stored in type {2} where I was expecting a list"
.format(dep_var._name, path_, type(dep_qualifiers)),
verbose_level=0)
else:
for entry in dep_qualifiers:
if type(entry) is dict:
name, value = entry.get("name",
None), str(entry.get("value", ""))
if name: dep_var._qualifiers.append((name, value))
for entry in dep_var_.get("values", []):
try:
val = float(entry["value"])
except ValueError as exc:
msg.error(
"HEP_data_helpers.get_dependent_variable_from_yaml_map",
"ValueError ({0}) when loading dependent variable {1} from file {2}"
.format(exc, dep_var._name, path_),
verbose_level=1)
val = None
except KeyError as exc:
msg.error(
"HEP_data_helpers.get_dependent_variable_from_yaml_map",
"KeyError ({0}) when loading dependent variable {1} from file {2}"
.format(exc, dep_var._name, path_),
verbose_level=1)
val = None
dep_var._values = np.append(dep_var._values, val)
pt_idx = 0
for entry in dep_var_.get("values", []):
errors = entry.get("errors", [])
err_idx = 0
for error in errors:
get_error_from_yaml_map(dep_var, error, pt_idx, err_idx)
err_idx = err_idx + 1
pt_idx = pt_idx + 1
return dep_var
def plot_ratio(self, key_num_, key_den_, **kwargs):
table_num = self._1D_distributions.get(key_num_, None)
if not table_num:
msg.error(
"HEP_data_utils.data_structures.DistributionContainer.plot_ratio",
"Error when plotting 1D distribution with key {0}... skipping".
format(key_num_))
raise KeyError("key {0} not in {1}".format(key_num_, self._name))
table_den = self._1D_distributions.get(key_den_, None)
if not table_den:
msg.error(
"HEP_data_utils.data_structures.DistributionContainer.plot_ratio",
"Error when plotting 1D distribution with key {0}... skipping".
format(key_den_))
raise KeyError("key {0} not in {1}".format(key_den_, self._name))
try:
plotter.plot_ratio(table_num, table_den, **kwargs)
except Exception as e:
print(e)
msg.error(
"HEP_data_utils.data_structures.DistributionContainer.plot_ratio",
"Error when plotting {0} / {1} ratio... skipping".format(
key_num_, key_den_))
def parse_inputs(argv_):
# Get arguments
try:
opts, rest = getopt.getopt(argv_, "hrps:t:v:", [
"help", "recursive", "default-2D-bins", "print", "show", "save=",
"type=", "verbosity="
])
except getopt.GetoptError as err:
msg.error(
"inspect_yaml.py",
"The following error was thrown whilst parsing command-line arguments"
)
print(">>>>>>>>\n", err, "\n<<<<<<<<")
msg.error("inspect_yaml.py", "Falling back to to --help...")
print_help()
msg.fatal("inspect_yaml.py", "Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
do_print_all = False
do_not_make_matrix = False
do_show = False
save_file = ""
restrict_type = None
for opt, arg in opts:
if opt in ['-h', "--help"]:
print_help()
sys.exit(0)
if opt in [
'-r',
"--recursive",
]:
msg.info("inspect_yaml.py",
"Config: using recursion if needed",
verbose_level=0)
do_recurse = True
if opt in ["--default-2D-bins"]:
msg.info(
"inspect_yaml.py",
"Config: I will *not* try to convert 2D binning into matrix format",
verbose_level=0)
do_not_make_matrix = True
if opt in ["--print"]:
msg.info("inspect_yaml.py",
"Config: printing all distributions found",
verbose_level=0)
do_print_all = True
if opt in ["--show"]:
msg.info("inspect_yaml.py",
"Config: showing all distributions found",
verbose_level=0)
do_show = True
if opt in ['-s', "--save"]:
save_file = str(arg)
if save_file[-4:] != ".pdf": save_file = save_file + ".pdf"
msg.info("inspect_yaml.py",
"Config: saving plots to {0}".format(save_file),
verbose_level=0)
if opt in ['-t', "--type"]:
arg = str(arg)
if arg not in ["root", "yaml"]:
msg.error(
"inspect_yaml.py",
"{0} option {1} not allowed: allowed inputs are \"root\" or \"yaml\" (deafult is both)"
)
else:
restrict_type = arg
msg.info("inspect_yaml.py",
"Config: only reading files of type {0}".format(
restrict_type),
verbose_level=0)
if opt in ['-v', "--verbosity"]:
msg.info("inspect_yaml.py",
"Config: setting verbosity to {0}".format(arg),
verbose_level=0)
try:
msg.VERBOSE_LEVEL = int(arg)
except:
msg.fatal(
"inspect_yaml.py",
"Could not cast verbosity level {0} to integer".format(
arg))
# Check that the remaining argument is valid
if len(rest) == 0:
msg.error("inspect_yaml.py", "No argument provided")
print_help()
msg.fatal("inspect_yaml.py",
"No input yaml file or directory provided")
if len(rest) == 1 and hlp.is_directory(rest[0]):
msg.info("inspect_yaml.py",
"Opening input directory {0}...".format(rest[0]),
verbose_level=0)
rest = [rest[0] + "/" + f for f in os.listdir(rest[0])]
yaml_files = []
if restrict_type == None or restrict_type == "yaml":
yaml_files = hlp.keep_only_yaml_files(rest, recurse=do_recurse)
root_files = []
if restrict_type == None or restrict_type == "root":
root_files = hlp.keep_only_root_files(rest, recurse=do_recurse)
if len(yaml_files + root_files) == 0:
msg.fatal(
"inspect_yaml.py",
"No input yaml or root files found from the inputs provided")
for f in rest:
msg.info("inspect_yaml.py",
"Registered input file {0}".format(f),
verbose_level=0)
# Return
return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file
def plot_ratio_1D(table_num_, table_den_, **kwargs):
x_n, y_n, [ex_lo_n, ex_hi_n], [
ey_lo_n, ey_hi_n
], labels, keys_num = plotter.get_1D_distribution(table_num_)
x_d, y_d, [ex_lo_d, ex_hi_d], [
ey_lo_d, ey_hi_d
], labels, keys_den = plotter.get_1D_distribution(table_den_)
chi2 = hlp.get_chi2(y_n, ey_lo_n, ey_hi_n, y_d, ey_lo_d, ey_hi_d)
for i in range(len(x_n)):
if x_n[i] == x_d[i]: continue
msg.error("plot_ratio_1D", "Arguments do not have the same binning")
raise ValueError(
"Ratio of distributions with bin centres at {0} and {1}", x_n, x_d)
fig = plotter.plt.figure(figsize=(10, 10))
ax1 = fig.add_subplot(211)
legend_char_width = 53
str_num_legend = "TABLE 1 ( " + " + ".join(keys_num) + " )"
str_num_legend = "\n".join([
str_num_legend[legend_char_width *
i:min(len(str_num_legend), legend_char_width * (i + 1))]
for i in range(int(len(str_num_legend) / legend_char_width) + 1)
])
str_den_legend = "TABLE 2 ( " + " + ".join(keys_den) + " )"
str_den_legend = "\n".join([
str_den_legend[legend_char_width *
i:min(len(str_den_legend), legend_char_width * (i + 1))]
for i in range(int(len(str_den_legend) / legend_char_width) + 1)
])
ax1.errorbar(x_n,
y_n,
yerr=[ey_lo_n, ey_hi_n],
xerr=[ex_lo_n, ex_hi_n],
c='k',
linewidth=2,
linestyle='None',
alpha=0.8,
label=str_num_legend)
ax1.errorbar(x_d,
y_d,
yerr=[ey_lo_d, ey_hi_d],
xerr=[ex_lo_d, ex_hi_d],
c='r',
linewidth=4,
linestyle='None',
alpha=0.4,
label=str_den_legend)
ax1.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plotter.plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4)
plotter.plt.ylabel("Values")
plotter.plt.ylabel("Values")
xlim = kwargs.get(
"xlim", [x_d[0] - np.fabs(ex_lo_n[0]), x_d[-1] + np.fabs(ex_hi_n[-1])])
ymin, ymax = ax1.get_ylim()
ax1.axis(xmin=xlim[0], xmax=xlim[1])
try:
plotter.plt.text(
xlim[0], 1.19 * ymax - 0.19 * ymin,
("$\\bf{TABLE}$ $\\bf{1:}$ " + table_num_._dep_var._name).replace(
"\\\\", "\\").replace(r"\text{", r"{\rm "))
plotter.plt.text(
xlim[0], 1.08 * ymax - 0.08 * ymin,
("$\\bf{TABLE}$ $\\bf{2:}$ " + table_den_._dep_var._name).replace(
"\\\\", "\\").replace(r"\text{", r"{\rm "))
except:
msg.warning(
"plot_ratio_1D",
"could not render observable name - no title given to plot")
plotter.plt.grid()
ax2 = fig.add_subplot(212)
ax2.errorbar(x_n,
y_n / y_d,
yerr=[ey_lo_n / y_d, ey_hi_n / y_d],
xerr=[ex_lo_n, ex_hi_n],
c='k',
linewidth=2,
linestyle='None',
alpha=0.8)
ax2.errorbar(x_d,
y_d / y_d,
yerr=[ey_lo_d / y_d, ey_hi_d / y_d],
xerr=[ex_lo_d, ex_hi_d],
c='r',
linewidth=4,
linestyle='None',
alpha=0.4)
box = ax2.get_position()
ax2.set_position([box.x0, box.y0, box.width * 0.4, box.height])
ax2.axis(xmin=xlim[0], xmax=xlim[1])
plotter.plt.ylabel("Ratio $\\bf{vs.}$ TABLE 2")
try:
plotter.plt.xlabel(
kwargs.get(
"xlabel", table_den_._indep_vars[0].name().replace(
"\\\\", "\\").replace(r"\text{", r"{\rm ")))
except:
plotter.plt.xlabel("<error reading xlabel")
plotter.plt.subplots_adjust(left=0.1, right=0.5, top=0.92, bottom=0.4)
plotter.plt.grid()
if kwargs.get("show", False):
plotter.plt.show()
if kwargs.get("save", False):
fig.savefig(plotter.document, format='pdf')
plotter.plt.close(fig)
return chi2, y_n / y_d
def parse_inputs(argv_):
# Get arguments
try:
opts, rest = getopt.getopt(argv_, "hrv:s:", [
"help", "recursive", "show", "save=", "verbosity=", "num=", "den="
])
except getopt.GetoptError as err:
msg.error(
"plot_ratios.py",
"The following error was thrown whilst parsing command-line arguments"
)
print(">>>>>>>>\n", err, "\n<<<<<<<<")
msg.error("plot_ratios.py", "Falling back to to --help...")
print_help()
msg.fatal("plot_ratios.py", "Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
do_show = False
save_file = ""
num_tag, den_tag = None, None
for opt, arg in opts:
if opt in ['-h', "--help"]:
print_help()
sys.exit(0)
if opt in [
'-r',
"--recursive",
]:
msg.info("plot_ratios.py",
"Config: using recursion if needed",
verbose_level=0)
do_recurse = True
if opt in [
"--num",
]:
num_tag = str(arg)
msg.info("plot_ratios.py",
"Config: numerators will be identified using the tag {0}".
format(num_tag),
verbose_level=0)
if opt in [
"--den",
]:
den_tag = str(arg)
msg.info(
"plot_ratios.py",
"Config: denominators will be identified using the tag {0}".
format(den_tag),
verbose_level=0)
if opt in ["--show"]:
msg.info("plot_contents_of_yaml.py",
"Config: showing all distributions found",
verbose_level=0)
do_show = True
if opt in ['-s', "--save"]:
save_file = str(arg)
if save_file[-4:] != ".pdf": save_file = save_file + ".pdf"
msg.info("plot_contents_of_yaml.py",
"Config: saving plots to {0}".format(save_file),
verbose_level=0)
if opt in ['-v', "--verbosity"]:
msg.info("plot_ratios.py",
"Config: setting verbosity to {0}".format(arg),
verbose_level=0)
try:
msg.VERBOSE_LEVEL = int(arg)
except:
msg.fatal(
"plot_ratios.py",
"Could not cast verbosity level {0} to integer".format(
arg))
yaml_files = hlp.keep_only_yaml_files(argv_, recurse=do_recurse)
root_files = hlp.keep_only_root_files(argv_, recurse=do_recurse)
if num_tag is None:
num_tag = "measured"
msg.warning(
"plot_ratios.py",
"No --num provided, falling back to \"{0}\"".format(num_tag))
if den_tag is None:
den_tag = "expected"
msg.warning(
"plot_ratios.py",
"No --den provided, falling back to \"{0}\"".format(den_tag))
# Return
return num_tag, den_tag, do_show, save_file, yaml_files, root_files
)
print("=== PLOTTING THE FOLLOWING RATIO ( 1 divided by 2 )")
print("=== 1. ", num_dist._dep_var._name)
print("=== 2. ", den_dist._dep_var._name)
try:
chi2, bin_ratios = plot_ratio(num_dist,
den_dist,
show=do_show,
save=do_save)
print("=== chi2 = {:.4f}".format(chi2))
print("=== bin ratios = ",
["{:.4f}".format(x) for x in bin_ratios])
except Exception as e:
print(e)
msg.error(
"HEP_data_utils.data_structures.DistributionContainer.plot_ratio",
"Error when plotting ratio... skipping")
print(
"====================================================================================="
)
#
# Close save file
#
if do_save: plotter.close_save_file()
#
# Goodbye
#
msg.info(
"plot_ratio_between_distributions.py",
"Program reached the end without crashing and will close :) Have a nice day..."
)
