def load_yaml_file(dataset_, path_, **kwargs):
msg.info("HEP_data_helpers.load_yaml_file",
"Opening yaml file {0}".format(path_),
verbose_level=0)
data = open_yaml_file(path_)
if len(data) != 1:
msg.error(
"HEP_data_helpers.load_yaml_file",
"{0} contains {1} tables where I was expecting 1... is the file format correct?"
.format(path_, len(data)))
return
dep_vars = data[0].get("dependent_variables", None)
if dep_vars is None:
msg.fatal(
"HEP_data_helpers.load_yaml_file",
"{0} contains no dependent_variables as required".format(path_))
return
indep_vars = data[0].get("independent_variables", None)
if indep_vars is None:
msg.fatal(
"HEP_data_helpers.load_yaml_file",
"{0} contains no independent_variables as required".format(path_))
return
load_distributions_from_yaml(dataset_, dep_vars, indep_vars, path_,
**kwargs)
def parse_inputs ( argv_ ) :
# Get arguments
try :
opts, rest = getopt.getopt(argv_,"hrv:",["help","recursive","verbosity=","yaml=","compare=",""])
except getopt.GetoptError as err :
msg.error("validate_yaml_files.py","The following error was thrown whilst parsing command-line arguments")
print(">>>>>>>>\n",err,"\n<<<<<<<<")
msg.error("validate_yaml_files.py","Falling back to to --help...")
print_help()
msg.fatal("validate_yaml_files.py","Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
for opt, arg in opts:
if opt in ['-h',"--help"] :
print_help()
sys.exit(0)
if opt in ['-r',"--recursive",] :
msg.info("validate_yaml_files.py","Config: using recursion if needed",verbose_level=0)
do_recurse = True
if opt in ['-v',"--verbosity"] :
msg.info("validate_yaml_files.py","Config: setting verbosity to {0}".format(arg),verbose_level=0)
try : msg.VERBOSE_LEVEL = int(arg)
except : msg.fatal("validate_yaml_files.py","Could not cast verbosity level {0} to integer".format(arg))
yaml_files = hlp.keep_only_yaml_files(get_argument_list(argv_,"--yaml"),recurse=do_recurse)
root_files = hlp.keep_only_root_files(get_argument_list(argv_,"--compare"),recurse=do_recurse)
if len(yaml_files) == 0 :
msg.fatal("validate_yaml_files.py","Please provide at least one yaml file using the --yaml option")
if len(root_files) == 0 :
msg.fatal("validate_yaml_files.py","Please provide at least one root file using the --compare option")
# Return
return yaml_files, root_files
def open_yaml_file(path_):
yaml_file = open(path_, 'r')
data = []
try:
for datum in yaml.safe_load_all(yaml_file):
msg.info("HEP_data_helpers.open_yaml_file",
"yaml file opened with entries:",
verbose_level=2)
msg.check_verbosity_and_print(yaml.safe_dump(datum),
verbose_level=2)
data.append(datum)
except yaml.YAMLError as exc:
print(exc)
msg.fatal(
"HEP_data_helpers.open_yaml_file",
"Exception thrown when opening the yaml file (see previous messages)"
)
return data
def get_argument_list ( argv , options ) :
if type(options) is not list : options = [options]
options = [ str(opt) for opt in options ]
options = [ a for a in argv if a in options ]
if len(options) > 1 :
msg.fatal("validate_yaml_files.py","Program arguments {0} mean the same thing - please only provide one".format(options))
arguments = []
save_arg = False
for el in argv :
if el in options :
save_arg = True
continue
if el[:1] is "-" :
save_arg = False
continue
if el is "validate_yaml_files.py" :
save_arg = False
continue
if not save_arg : continue
arguments.append(el)
return arguments
def get_2D_plottable_bins(table_):
if str(type(table_)
) != "<class 'HEP_data_utils.data_structures.HEPDataTable'>":
msg.fatal("HEP_data_utils.plotting.get_2D_plottable_bins",
"argument must be of type HEPDataTable")
dep_var, indep_vars = table_._dep_var, table_._indep_vars
if len(indep_vars) != 2:
msg.fatal(
"HEP_data_utils.plotting.get_2D_plottable_bins",
"HEPDataTable {0} has {1} independent_variable where 2 are required"
.format(dep_var.name(), len(indep_vars)))
values, n_vals = dep_var._values, len(dep_var)
old_bin_labels_x, old_bin_labels_y = indep_vars[0]._bin_labels, indep_vars[
1]._bin_labels
use_labels_x, use_labels_y = True, True
for label in old_bin_labels_x:
if len(label) == 0: use_labels_x = False
for label in old_bin_labels_y:
if len(label) == 0: use_labels_y = False
if not use_labels_x:
for i in range(len(old_bin_labels_x)):
old_bin_labels_x[i] = "{0:.2f}[{1:.2f},{2:.2f}]".format(
indep_vars[0]._bin_centers[i], indep_vars[0]._bin_centers[i] -
indep_vars[0]._bin_widths_lo[i],
indep_vars[0]._bin_centers[i] +
indep_vars[0]._bin_widths_hi[i])
if not use_labels_y:
for i in range(len(old_bin_labels_y)):
old_bin_labels_y[i] = "{0:.2f}[{1:.2f},{2:.2f}]".format(
indep_vars[1]._bin_centers[i], indep_vars[1]._bin_centers[i] -
indep_vars[1]._bin_widths_lo[i],
indep_vars[1]._bin_centers[i] +
indep_vars[1]._bin_widths_hi[i])
old_n_bins_x = len(old_bin_labels_x)
old_n_bins_y = len(old_bin_labels_y)
if values.shape == (old_n_bins_x, old_n_bins_y):
return old_bin_labels_x, old_bin_labels_y, values
if values.shape == (old_n_bins_y, old_n_bins_x):
return old_bin_labels_x, old_bin_labels_y, values.T()
if n_vals == old_n_bins_x == old_n_bins_y:
bin_labels_x = [y for y in {x for x in old_bin_labels_x}]
bin_labels_x = natsorted(bin_labels_x, alg=ns.IGNORECASE)
bin_labels_y = [y for y in {x for x in old_bin_labels_y}]
bin_labels_y = natsorted(bin_labels_y, alg=ns.IGNORECASE)
new_n_bins_x = len(bin_labels_x)
new_n_bins_y = len(bin_labels_y)
new_values = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x, y, v in zip(old_bin_labels_x, old_bin_labels_y, values):
new_values[bin_labels_x.index(x), bin_labels_y.index(y)] = v
return bin_labels_x, bin_labels_y, new_values
if n_vals == old_n_bins_x * old_n_bins_y:
new_values = np.array(np.zeros(shape=(old_n_bins_x, old_n_bins_y)))
for x_idx in enumerate(old_bin_labels_x):
for y_idx in enumerate(old_bin_labels_y):
v = values[x_idx + old_n_bins_x * y_idx]
new_values[x_idx, y_idx] = v
return old_bin_labels_x, old_bin_labels_y, new_values
msg.fatal("HEP_data_utils.plotting.get_2D_plottable_bins",
"HEPDataTable {0} is not a valid matrix".format(dep_var.name()))
def parse_inputs(argv_):
# Get arguments
try:
opts, rest = getopt.getopt(argv_, "hrps:t:v:", [
"help", "recursive", "default-2D-bins", "print", "show", "save=",
"type=", "verbosity="
])
except getopt.GetoptError as err:
msg.error(
"inspect_yaml.py",
"The following error was thrown whilst parsing command-line arguments"
)
print(">>>>>>>>\n", err, "\n<<<<<<<<")
msg.error("inspect_yaml.py", "Falling back to to --help...")
print_help()
msg.fatal("inspect_yaml.py", "Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
do_print_all = False
do_not_make_matrix = False
do_show = False
save_file = ""
restrict_type = None
for opt, arg in opts:
if opt in ['-h', "--help"]:
print_help()
sys.exit(0)
if opt in [
'-r',
"--recursive",
]:
msg.info("inspect_yaml.py",
"Config: using recursion if needed",
verbose_level=0)
do_recurse = True
if opt in ["--default-2D-bins"]:
msg.info(
"inspect_yaml.py",
"Config: I will *not* try to convert 2D binning into matrix format",
verbose_level=0)
do_not_make_matrix = True
if opt in ["--print"]:
msg.info("inspect_yaml.py",
"Config: printing all distributions found",
verbose_level=0)
do_print_all = True
if opt in ["--show"]:
msg.info("inspect_yaml.py",
"Config: showing all distributions found",
verbose_level=0)
do_show = True
if opt in ['-s', "--save"]:
save_file = str(arg)
if save_file[-4:] != ".pdf": save_file = save_file + ".pdf"
msg.info("inspect_yaml.py",
"Config: saving plots to {0}".format(save_file),
verbose_level=0)
if opt in ['-t', "--type"]:
arg = str(arg)
if arg not in ["root", "yaml"]:
msg.error(
"inspect_yaml.py",
"{0} option {1} not allowed: allowed inputs are \"root\" or \"yaml\" (deafult is both)"
)
else:
restrict_type = arg
msg.info("inspect_yaml.py",
"Config: only reading files of type {0}".format(
restrict_type),
verbose_level=0)
if opt in ['-v', "--verbosity"]:
msg.info("inspect_yaml.py",
"Config: setting verbosity to {0}".format(arg),
verbose_level=0)
try:
msg.VERBOSE_LEVEL = int(arg)
except:
msg.fatal(
"inspect_yaml.py",
"Could not cast verbosity level {0} to integer".format(
arg))
# Check that the remaining argument is valid
if len(rest) == 0:
msg.error("inspect_yaml.py", "No argument provided")
print_help()
msg.fatal("inspect_yaml.py",
"No input yaml file or directory provided")
if len(rest) == 1 and hlp.is_directory(rest[0]):
msg.info("inspect_yaml.py",
"Opening input directory {0}...".format(rest[0]),
verbose_level=0)
rest = [rest[0] + "/" + f for f in os.listdir(rest[0])]
yaml_files = []
if restrict_type == None or restrict_type == "yaml":
yaml_files = hlp.keep_only_yaml_files(rest, recurse=do_recurse)
root_files = []
if restrict_type == None or restrict_type == "root":
root_files = hlp.keep_only_root_files(rest, recurse=do_recurse)
if len(yaml_files + root_files) == 0:
msg.fatal(
"inspect_yaml.py",
"No input yaml or root files found from the inputs provided")
for f in rest:
msg.info("inspect_yaml.py",
"Registered input file {0}".format(f),
verbose_level=0)
# Return
return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file
def reformat_2D_bins_as_matrix_using_labels(dist_2D_):
if type(dist_2D_) != HEPDataTable:
msg.fatal("HEP_data_helpers.reformat_2D_bins_as_matrix_using_labels",
"argument must be of type HEPDataTable")
dep_var = dist_2D_._dep_var
if len(dist_2D_._indep_vars) != 2:
msg.fatal(
"HEP_data_helpers.reformat_2D_bins_as_matrix_using_labels",
"HEPDataTable {0} has {1} independent_variable where 2 are required"
.format(dep_var._name, len(dist_2D_._indep_vars)))
n_vals = len(dep_var)
old_bin_labels_x = dist_2D_._indep_vars[0]._bin_labels
old_bin_labels_y = dist_2D_._indep_vars[1]._bin_labels
old_n_bins_x = len(old_bin_labels_x)
old_n_bins_y = len(old_bin_labels_y)
if dep_var._values.shape == (old_bin_labels_x, old_bin_labels_y):
return
if n_vals == old_n_bins_x == old_n_bins_y:
bin_labels_x = [y for y in {x for x in old_bin_labels_x}]
bin_labels_x = natsorted(bin_labels_x, alg=ns.IGNORECASE)
bin_labels_y = [y for y in {x for x in old_bin_labels_y}]
bin_labels_y = natsorted(bin_labels_y, alg=ns.IGNORECASE)
new_n_bins_x = len(bin_labels_x)
new_n_bins_y = len(bin_labels_y)
new_values = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x, y, v in zip(old_bin_labels_x, old_bin_labels_y,
dep_var._values):
new_values[bin_labels_x.index(x), bin_labels_y.index(y)] = v
dep_var._values = new_values
for key, values in dep_var._symerrors.items():
new_error = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x, y, v in zip(old_bin_labels_x, old_bin_labels_y, values):
new_error[bin_labels_x.index(x), bin_labels_y.index(y)] = v
dep_var._symerrors[key] = new_error
for key, values in dep_var._asymerrors_up.items():
new_error = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x, y, v in zip(old_bin_labels_x, old_bin_labels_y, values):
new_error[bin_labels_x.index(x), bin_labels_y.index(y)] = v
dep_var._asymerrors_up[key] = new_error
for key, values in dep_var._asymerrors_dn.items():
new_error = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x, y, v in zip(old_bin_labels_x, old_bin_labels_y, values):
new_error[bin_labels_x.index(x), bin_labels_y.index(y)] = v
dep_var._asymerrors_dn[key] = new_error
dist_2D_._indep_vars[0].set_bin_labels(bin_labels_x)
dist_2D_._indep_vars[1].set_bin_labels(bin_labels_y)
return
if n_vals == old_n_bins_x * old_n_bins_y:
new_values = np.array(np.zeros(shape=(old_n_bins_x, old_n_bins_y)))
print(dep_var._values)
print(dep_var._values.shape)
for x_idx in range(len(old_bin_labels_x)):
for y_idx in range(len(old_bin_labels_y)):
new_values[x_idx,
y_idx] = dep_var._values[x_idx +
old_n_bins_x * y_idx]
dep_var._values = new_values
for key, values in dist_2D_._symerrors.items():
new_error = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x_idx in range(len(old_bin_labels_x)):
for y_idx in range(len(old_bin_labels_y)):
new_error[x_idx,
y_idx] = values[x_idx + old_n_bins_x * y_idx]
dep_var._symerrors[key] = new_error
for key, values in dist_2D_._asymerrors_up.items():
new_error = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x_idx in range(len(old_bin_labels_x)):
for y_idx in range(len(old_bin_labels_y)):
new_error[x_idx,
y_idx] = values[x_idx + old_n_bins_x * y_idx]
dep_var._asymerrors_up[key] = new_error
for key, values in dist_2D_._asymerrors_dn.items():
new_error = np.array(np.zeros(shape=(new_n_bins_x, new_n_bins_y)))
for x_idx in range(len(old_bin_labels_x)):
for y_idx in range(len(old_bin_labels_y)):
new_error[x_idx,
y_idx] = values[x_idx + old_n_bins_x * y_idx]
dep_var._asymerrors_dn[key] = new_error
return
msg.warning(
"HEP_data_helpers.reformat_2D_bins_as_matrix_using_labels",
"HEPDataTable {0} is not a matrix... assuming it is a non-factorisable distribution and returning with nothing done"
.format(dep_var._name),
verbose_level=0)
return
def parse_inputs(argv_):
# Get arguments
try:
opts, rest = getopt.getopt(argv_, "hrv:s:", [
"help", "recursive", "show", "save=", "verbosity=", "num=", "den="
])
except getopt.GetoptError as err:
msg.error(
"plot_ratios.py",
"The following error was thrown whilst parsing command-line arguments"
)
print(">>>>>>>>\n", err, "\n<<<<<<<<")
msg.error("plot_ratios.py", "Falling back to to --help...")
print_help()
msg.fatal("plot_ratios.py", "Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
do_show = False
save_file = ""
num_tag, den_tag = None, None
for opt, arg in opts:
if opt in ['-h', "--help"]:
print_help()
sys.exit(0)
if opt in [
'-r',
"--recursive",
]:
msg.info("plot_ratios.py",
"Config: using recursion if needed",
verbose_level=0)
do_recurse = True
if opt in [
"--num",
]:
num_tag = str(arg)
msg.info("plot_ratios.py",
"Config: numerators will be identified using the tag {0}".
format(num_tag),
verbose_level=0)
if opt in [
"--den",
]:
den_tag = str(arg)
msg.info(
"plot_ratios.py",
"Config: denominators will be identified using the tag {0}".
format(den_tag),
verbose_level=0)
if opt in ["--show"]:
msg.info("plot_contents_of_yaml.py",
"Config: showing all distributions found",
verbose_level=0)
do_show = True
if opt in ['-s', "--save"]:
save_file = str(arg)
if save_file[-4:] != ".pdf": save_file = save_file + ".pdf"
msg.info("plot_contents_of_yaml.py",
"Config: saving plots to {0}".format(save_file),
verbose_level=0)
if opt in ['-v', "--verbosity"]:
msg.info("plot_ratios.py",
"Config: setting verbosity to {0}".format(arg),
verbose_level=0)
try:
msg.VERBOSE_LEVEL = int(arg)
except:
msg.fatal(
"plot_ratios.py",
"Could not cast verbosity level {0} to integer".format(
arg))
yaml_files = hlp.keep_only_yaml_files(argv_, recurse=do_recurse)
root_files = hlp.keep_only_root_files(argv_, recurse=do_recurse)
if num_tag is None:
num_tag = "measured"
msg.warning(
"plot_ratios.py",
"No --num provided, falling back to \"{0}\"".format(num_tag))
if den_tag is None:
den_tag = "expected"
msg.warning(
"plot_ratios.py",
"No --den provided, falling back to \"{0}\"".format(den_tag))
# Return
return num_tag, den_tag, do_show, save_file, yaml_files, root_files