def get_uproot_histograms(data_):
ret = {}
for key, entry in data_.items():
if str(type(entry)) == "<class 'uproot.rootio.TH1F'>":
ret[key] = get_ROOT_Table_from_uproot_TH1(entry)
elif str(type(entry)) == "<class 'uproot.rootio.TH1D'>":
ret[key] = get_ROOT_Table_from_uproot_TH1(entry)
elif str(type(entry)) == "<class 'uproot.rootio.TH2F'>":
ret[key] = get_ROOT_Table_from_uproot_TH2(entry)
elif str(type(entry)) == "<class 'uproot.rootio.TH2D'>":
ret[key] = get_ROOT_Table_from_uproot_TH2(entry)
elif str(type(entry)) == "<class 'uproot.rootio.TGraph'>":
ret[key] = get_ROOT_Table_from_uproot_TGraph(entry)
elif str(type(entry)) == "<class 'uproot.rootio.TGraphErrors'>":
ret[key] = get_ROOT_Table_from_uproot_TGraphErrors(entry)
elif str(type(entry)) == "<class 'uproot.rootio.TGraphAsymmErrors'>":
ret[key] = get_ROOT_Table_from_uproot_TGraphAsymmErrors(entry)
elif str(type(entry)) == "<class 'uproot.rootio.TGraph2DErrors'>":
ret[key] = get_ROOT_Table_from_uproot_TGraph2DErrors(entry)
else:
msg.info(
"HEP_data_utils.ROOT_helpers.get_uproot_histograms",
"Object {0} of type {1} is ignored as I don't understand this format"
.format(key, type(entry)),
verbose_level=0)
return ret
def load_yaml_file(dataset_, path_, **kwargs):
msg.info("HEP_data_helpers.load_yaml_file",
"Opening yaml file {0}".format(path_),
verbose_level=0)
data = open_yaml_file(path_)
if len(data) != 1:
msg.error(
"HEP_data_helpers.load_yaml_file",
"{0} contains {1} tables where I was expecting 1... is the file format correct?"
.format(path_, len(data)))
return
dep_vars = data[0].get("dependent_variables", None)
if dep_vars is None:
msg.fatal(
"HEP_data_helpers.load_yaml_file",
"{0} contains no dependent_variables as required".format(path_))
return
indep_vars = data[0].get("independent_variables", None)
if indep_vars is None:
msg.fatal(
"HEP_data_helpers.load_yaml_file",
"{0} contains no independent_variables as required".format(path_))
return
load_distributions_from_yaml(dataset_, dep_vars, indep_vars, path_,
**kwargs)
def parse_inputs ( argv_ ) :
# Get arguments
try :
opts, rest = getopt.getopt(argv_,"hrv:",["help","recursive","verbosity=","yaml=","compare=",""])
except getopt.GetoptError as err :
msg.error("validate_yaml_files.py","The following error was thrown whilst parsing command-line arguments")
print(">>>>>>>>\n",err,"\n<<<<<<<<")
msg.error("validate_yaml_files.py","Falling back to to --help...")
print_help()
msg.fatal("validate_yaml_files.py","Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
for opt, arg in opts:
if opt in ['-h',"--help"] :
print_help()
sys.exit(0)
if opt in ['-r',"--recursive",] :
msg.info("validate_yaml_files.py","Config: using recursion if needed",verbose_level=0)
do_recurse = True
if opt in ['-v',"--verbosity"] :
msg.info("validate_yaml_files.py","Config: setting verbosity to {0}".format(arg),verbose_level=0)
try : msg.VERBOSE_LEVEL = int(arg)
except : msg.fatal("validate_yaml_files.py","Could not cast verbosity level {0} to integer".format(arg))
yaml_files = hlp.keep_only_yaml_files(get_argument_list(argv_,"--yaml"),recurse=do_recurse)
root_files = hlp.keep_only_root_files(get_argument_list(argv_,"--compare"),recurse=do_recurse)
if len(yaml_files) == 0 :
msg.fatal("validate_yaml_files.py","Please provide at least one yaml file using the --yaml option")
if len(root_files) == 0 :
msg.fatal("validate_yaml_files.py","Please provide at least one root file using the --compare option")
# Return
return yaml_files, root_files
def get_error_from_yaml_map(dep_var_, error_, pt_idx_, err_idx_=0):
key = error_.get("label", "err{0}".format(err_idx_))
if "symerror" in error_:
if key not in dep_var_._symerrors:
msg.info("HEP_data_helpers.get_error_from_yaml_map",
"Creating symmetric error {0} with length {1}".format(
key, len(dep_var_)),
verbose_level=2)
dep_var_._symerrors[key] = np.zeros(shape=(len(dep_var_)))
dep_var_._symerrors[key][pt_idx_] = float(error_["symerror"])
elif "asymerror" in error_:
err_asymm = error_["asymerror"]
if key not in dep_var_._asymerrors_up:
msg.info("HEP_data_helpers.get_error_from_yaml_map",
"Creating asymmetric error {0} with length {1}".format(
key, len(dep_var_)),
verbose_level=2)
dep_var_._asymerrors_up[key] = np.zeros(shape=(len(dep_var_)))
dep_var_._asymerrors_dn[key] = np.zeros(shape=(len(dep_var_)))
if "plus" not in err_asymm:
msg.error("HEP_data_helpers.get_error_from_yaml_map",
"No entry named \"plus\" for error \"asymerror\"")
else:
dep_var_._asymerrors_up[key][pt_idx_] = float(err_asymm["plus"])
if "minus" not in err_asymm:
msg.error("HEP_data_helpers.get_error_from_yaml_map",
"No entry named \"minus\" for error \"asymerror\"")
else:
dep_var_._asymerrors_dn[key][pt_idx_] = float(err_asymm["minus"])
else:
print(yaml.safe_dump(error_))
msg.error("HEP_data_helpers.get_error_from_yaml_map",
"map does not have an entry called symerror or asymerror")
return key
def load_root_file(dataset_, path_, **kwargs):
msg.info("ROOT_helpers.load_root_file",
"Opening root file {0}".format(path_),
verbose_level=0)
raw_uproot_data = open_root_file(path_, path_)
uproot_histos = get_uproot_histograms(raw_uproot_data)
for key, table in uproot_histos.items():
load_distribution_from_ROOT_Table(dataset_, key, table, **kwargs)
def set_save_file ( fname_ ) :
global document
if type(fname_) is str :
if type(document) is PdfPages : document.close()
if fname_[-4:] != ".pdf" : fname_ = fname_ + ".pdf"
msg.info("HEP_data_utils.plotting.set_save_file","Opening pdf file {0}".format(fname_),verbose_level=0)
document = PdfPages(fname_)
else : msg.error("HEP_data_utils.plotting.set_save_file","Filename must be a str")
def load_root_files_from_list ( dataset_ , dir_ , **kwargs ) :
if hlp.is_directory(dir_) :
for filename in [ dir_+"/"+f for f in os.listdir(dir_) if is_root_file(f) ] :
msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening root file {0}".format(filename),verbose_level=0)
load_all_root_files(dataset_,filename,**kwargs)
elif type(dir_) == list :
for filename in dir_ :
if type(filename) != str : continue
msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening yaml file {0}".format(filename),verbose_level=0)
load_all_root_files(dataset_,filename,**kwargs)
else :
msg.error("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Input {0} is neither a directory nor a list... returning with nothing done".format(dir_),verbose_level=-1)
def open_yaml_file(path_):
yaml_file = open(path_, 'r')
data = []
try:
for datum in yaml.safe_load_all(yaml_file):
msg.info("HEP_data_helpers.open_yaml_file",
"yaml file opened with entries:",
verbose_level=2)
msg.check_verbosity_and_print(yaml.safe_dump(datum),
verbose_level=2)
data.append(datum)
except yaml.YAMLError as exc:
print(exc)
msg.fatal(
"HEP_data_helpers.open_yaml_file",
"Exception thrown when opening the yaml file (see previous messages)"
)
return data
def rename_key(self, old_key_, new_key_):
something_done = False
old_key = r"{0}".format(old_key_)
new_key = r"{0}".format(new_key_)
for key in self._inclusive_distributions:
if old_key != key: continue
self._inclusive_distributions[
new_key_] = self._inclusive_distributions.pop(old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming inclusive distribution key {1} to {2}".
format(self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
for key in self._1D_distributions:
if old_key != key: continue
self._1D_distributions[new_key_] = self._1D_distributions.pop(
old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming 1D distribution key {1} to {2}".format(
self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
for key in self._2D_distributions:
if old_key != key: continue
self._2D_distributions[new_key_] = self._2D_distributions.pop(
old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming 2D distribution key {1} to {2}".format(
self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
for key in self._ND_distributions:
if old_key != key: continue
self._ND_distributions[new_key_] = self._ND_distributions.pop(
old_key_)
msg.info(
"DistributionContainer.rename_key",
"Store \"{0}\" renaming ND distribution key {1} to {2}".format(
self._name, old_key_, new_key_),
verbose_level=1)
something_done = True
if not something_done:
msg.error(
"DistributionContainer.rename_key",
"Store \"{0}\" with nothing done for old_key_={1}, new_key_={2}"
.format(self._name, old_key_, new_key_),
verbose_level=1)
for err in errs_not_match :
not_matches.append(err)
matches = list(set(matches))
not_matches = list({ x for x in not_matches if x not in matches })
print_match_result ( key_ , central_values_match , matches , not_matches )
# =================================== #
# ==== Brief: main program ==== #
# =================================== #
if __name__ == "__main__" :
#
# Welcome
#
msg.info("validate_yaml_files.py","Running program")
#
# Get input files and settings
#
yamls_to_load, roots_to_load = parse_inputs(sys.argv[1:])
#
# Load input files
#
yaml_tables = DistributionContainer("yaml_files")
HD.load_yaml_files_from_list(yaml_tables,yamls_to_load)
#
root_tables = DistributionContainer("root_files")
RT.load_root_files_from_list(root_tables,roots_to_load)
#
# Look for matches
#
except Exception as e:
print(e)
msg.error("plot_ratios", "Error when plotting ratio... skipping")
print(
"====================================================================================="
)
# =================================== #
# ==== Brief: main program ==== #
# =================================== #
if __name__ == "__main__":
#
# Welcome
#
msg.info("plot_ratios.py", "Running program")
#
# Get input files and settings
#
num_tag, den_tag, do_show, save_file, yamls_to_load, roots_to_load = parse_inputs(
sys.argv[1:])
do_save = len(save_file) > 0
if do_save: plotter.set_save_file(save_file)
#
# Load input files
#
my_tables = DistributionContainer("my_tables")
HD.load_yaml_files_from_list(my_tables, yamls_to_load)
RT.load_root_files_from_list(my_tables, roots_to_load)
#
# Get numerator and denominator tables
def parse_inputs(argv_):
# Get arguments
try:
opts, rest = getopt.getopt(argv_, "hrps:t:v:", [
"help", "recursive", "default-2D-bins", "print", "show", "save=",
"type=", "verbosity="
])
except getopt.GetoptError as err:
msg.error(
"inspect_yaml.py",
"The following error was thrown whilst parsing command-line arguments"
)
print(">>>>>>>>\n", err, "\n<<<<<<<<")
msg.error("inspect_yaml.py", "Falling back to to --help...")
print_help()
msg.fatal("inspect_yaml.py", "Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
do_print_all = False
do_not_make_matrix = False
do_show = False
save_file = ""
restrict_type = None
for opt, arg in opts:
if opt in ['-h', "--help"]:
print_help()
sys.exit(0)
if opt in [
'-r',
"--recursive",
]:
msg.info("inspect_yaml.py",
"Config: using recursion if needed",
verbose_level=0)
do_recurse = True
if opt in ["--default-2D-bins"]:
msg.info(
"inspect_yaml.py",
"Config: I will *not* try to convert 2D binning into matrix format",
verbose_level=0)
do_not_make_matrix = True
if opt in ["--print"]:
msg.info("inspect_yaml.py",
"Config: printing all distributions found",
verbose_level=0)
do_print_all = True
if opt in ["--show"]:
msg.info("inspect_yaml.py",
"Config: showing all distributions found",
verbose_level=0)
do_show = True
if opt in ['-s', "--save"]:
save_file = str(arg)
if save_file[-4:] != ".pdf": save_file = save_file + ".pdf"
msg.info("inspect_yaml.py",
"Config: saving plots to {0}".format(save_file),
verbose_level=0)
if opt in ['-t', "--type"]:
arg = str(arg)
if arg not in ["root", "yaml"]:
msg.error(
"inspect_yaml.py",
"{0} option {1} not allowed: allowed inputs are \"root\" or \"yaml\" (deafult is both)"
)
else:
restrict_type = arg
msg.info("inspect_yaml.py",
"Config: only reading files of type {0}".format(
restrict_type),
verbose_level=0)
if opt in ['-v', "--verbosity"]:
msg.info("inspect_yaml.py",
"Config: setting verbosity to {0}".format(arg),
verbose_level=0)
try:
msg.VERBOSE_LEVEL = int(arg)
except:
msg.fatal(
"inspect_yaml.py",
"Could not cast verbosity level {0} to integer".format(
arg))
# Check that the remaining argument is valid
if len(rest) == 0:
msg.error("inspect_yaml.py", "No argument provided")
print_help()
msg.fatal("inspect_yaml.py",
"No input yaml file or directory provided")
if len(rest) == 1 and hlp.is_directory(rest[0]):
msg.info("inspect_yaml.py",
"Opening input directory {0}...".format(rest[0]),
verbose_level=0)
rest = [rest[0] + "/" + f for f in os.listdir(rest[0])]
yaml_files = []
if restrict_type == None or restrict_type == "yaml":
yaml_files = hlp.keep_only_yaml_files(rest, recurse=do_recurse)
root_files = []
if restrict_type == None or restrict_type == "root":
root_files = hlp.keep_only_root_files(rest, recurse=do_recurse)
if len(yaml_files + root_files) == 0:
msg.fatal(
"inspect_yaml.py",
"No input yaml or root files found from the inputs provided")
for f in rest:
msg.info("inspect_yaml.py",
"Registered input file {0}".format(f),
verbose_level=0)
# Return
return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file
def print_help():
msg.info(
"inspect_yaml.py",
" Usage: python3 inspect_yaml.py <yaml file(s) OR directory OR submission.yaml file>"
)
msg.info(
"inspect_yaml.py",
" If input files/directory contains a submission.yaml file, all other inputs will be ignored"
)
msg.info("inspect_yaml.py",
" I assume that you follow the format instructions provided at")
msg.info(
"inspect_yaml.py",
" https://hepdata-submission.readthedocs.io/en/latest/introduction.html"
)
msg.info("inspect_yaml.py", " Optional arguments are:")
msg.info("inspect_yaml.py",
" -h, --help\t\tPrint this help message and close")
msg.info(
"inspect_yaml.py",
" -v, --verbosity\tSet VERBOSE_LEVEL {-1, 0, 1, 2} (-1 by default)"
)
msg.info(
"inspect_yaml.py",
" -r, --recursive\tAllow recursive searching of directories")
msg.info(
"inspect_yaml.py",
" \tRecursion stops if submission.yaml file is found"
)
msg.info("inspect_yaml.py",
" -s, --save\t\tSave plots to the file provided")
msg.info(
"inspect_yaml.py",
" -t, --type\t\tSpecify input type as { root , yaml } (default is both)"
)
msg.info(
"inspect_yaml.py",
" --print\t\tPrint all information on the distributions found")
msg.info("inspect_yaml.py",
" --show\t\t\tShow plots to the screen")
msg.info(
"inspect_yaml.py",
" --default-2D-bins\tPrevent interpretation of 2D bins as a matrix"
)
msg.info(
"inspect_yaml.py",
" \t(will be stored as a 1D vector instead)")
msg.info(
"inspect_yaml.py",
" N.B. you can validate your yaml file format using the package:")
msg.info("inspect_yaml.py",
" https://github.com/HEPData/hepdata-validator")
for f in rest:
msg.info("inspect_yaml.py",
"Registered input file {0}".format(f),
verbose_level=0)
# Return
return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file
# =================================== #
# ==== Brief: main program ==== #
# =================================== #
if __name__ == "__main__":
#
# Welcome
#
msg.info("inspect_yaml.py", "Running program")
#
# Get input files and settings
#
yamls_to_load, roots_to_load, do_show, do_print_all, do_not_make_matrix, save_file = parse_inputs(
sys.argv[1:])
do_save = len(save_file) > 0
if do_save: plotter.set_save_file(save_file)
if not do_show and not do_save and not do_print_all:
msg.warning(
"inspect_yaml.py",
"Neither --save, --show nor --print specified. Falling back to --print."
)
do_print_all = True
#
# Load input files
for f in rest:
msg.info("plot_contents_of_yaml.py",
"Registered input file {0}".format(f),
verbose_level=0)
# Return
return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file
# =================================== #
# ==== Brief: main program ==== #
# =================================== #
if __name__ == "__main__":
#
# Welcome
#
msg.info("plot_contents_of_yaml.py", "Running program")
#
# Get input files and settings
#
yamls_to_load, roots_to_load, do_show, do_print_all, do_not_make_matrix, save_file = parse_inputs(
sys.argv[1:])
do_save = len(save_file) > 0
if do_save: plotter.set_save_file(save_file)
if not do_show and not do_save and not do_print_all:
msg.warning(
"plot_contents_of_yaml.py",
"Neither --save, --show nor --print specified. Falling back to --print."
)
do_print_all = True
#
# Load input files
def print_help():
msg.info(
"plot_ratios.py",
"Usage: python3 plot_ratios.py --num <numerator tag> --den <denominator tag> <input-files>"
)
msg.info(
"plot_ratios.py",
"If input files/directory contains a submission.yaml file, all other inputs will be ignored"
)
msg.info("plot_ratios.py",
"I assume that you follow the format instructions provided at")
msg.info(
"plot_ratios.py",
" https://hepdata-submission.readthedocs.io/en/latest/introduction.html"
)
msg.info("plot_ratios.py", "Optional arguments are:")
msg.info("plot_ratios.py",
" -h, --help\tPrint this help message and close")
msg.info(
"plot_ratios.py",
" -v, --verbosity\tSet VERBOSE_LEVEL {-1, 0, 1, 2} (-1 by default)"
)
msg.info("plot_ratios.py",
" -r, --recursive\tAllow recursive searching of directories")
msg.info(
"plot_ratios.py",
" \tRecursion stops if submission.yaml file is found"
)
msg.info("plot_ratios.py",
" -s, --save\t\tSave plots to the file provided")
msg.info("plot_ratios.py", " --show\t\tShow plots to the screen")
msg.info("plot_ratios.py",
"N.B. you can validate your yaml file format using the package:")
msg.info("plot_ratios.py",
" https://github.com/HEPData/hepdata-validator")
def print_help () :
msg.info("validate_yaml_files.py:print_help","Usage is: python3 validate_yaml_files.py --yaml <yaml file(s)> --compare <root file(s)>")
msg.info("validate_yaml_files.py:print_help","If input files/directory contains a submission.yaml file, all other inputs will be ignored")
msg.info("validate_yaml_files.py:print_help","I assume that you follow the format instructions provided at https://hepdata-submission.readthedocs.io/en/latest/introduction.html")
msg.info("validate_yaml_files.py:print_help","Optional arguments are:")
msg.info("validate_yaml_files.py:print_help"," -h, --help\t\tPrint this help message and close")
msg.info("validate_yaml_files.py:print_help"," -v, --verbosity\t\tSet HEP_data_utils.messaging.VERBOSE_LEVEL as {-1, 0, 1, 2} (-1 by default)")
msg.info("validate_yaml_files.py:print_help"," -r, --recursive\t\tAllow recursive searching of directories. Recursion stops if submission.yaml file is found")
msg.info("validate_yaml_files.py:print_help","N.B. you can validate your yaml files using the following package: https://github.com/HEPData/hepdata-validator")
print(histo_.xlabel)
print(histo_.xvalues)
print(histo_.ylabel)
print(histo_.yvalues)
return
print([x for x in dir(histo_) if x[:1] != "_"])
# =================================== #
# ==== Brief: main program ==== #
# =================================== #
if __name__ == "__main__":
#
# Welcome
#
msg.info("test_uproot.py", "Running program")
#
# Get input files and settings
#
filename = "make_rootfile_output.root"
histograms = get_histograms(filename)
for key, histo in histograms.items():
print(key)
if str(type(histo)) == "<class 'uproot.rootio.TH1'>": read_TH1(histo)
elif str(type(histo)) == "<class 'uproot.rootio.TH1F'>":
read_TH1(histo)
elif str(type(histo)) == "<class 'uproot.rootio.TH1D'>":
read_TH1(histo)
elif str(type(histo)) == "<class 'uproot.rootio.TH2'>":
read_TH2(histo)
elif str(type(histo)) == "<class 'uproot.rootio.TH2F'>":
def parse_inputs(argv_):
# Get arguments
try:
opts, rest = getopt.getopt(argv_, "hrv:s:", [
"help", "recursive", "show", "save=", "verbosity=", "num=", "den="
])
except getopt.GetoptError as err:
msg.error(
"plot_ratios.py",
"The following error was thrown whilst parsing command-line arguments"
)
print(">>>>>>>>\n", err, "\n<<<<<<<<")
msg.error("plot_ratios.py", "Falling back to to --help...")
print_help()
msg.fatal("plot_ratios.py", "Command-line arguments not recognised.")
# Parse arguments
do_recurse = False
do_show = False
save_file = ""
num_tag, den_tag = None, None
for opt, arg in opts:
if opt in ['-h', "--help"]:
print_help()
sys.exit(0)
if opt in [
'-r',
"--recursive",
]:
msg.info("plot_ratios.py",
"Config: using recursion if needed",
verbose_level=0)
do_recurse = True
if opt in [
"--num",
]:
num_tag = str(arg)
msg.info("plot_ratios.py",
"Config: numerators will be identified using the tag {0}".
format(num_tag),
verbose_level=0)
if opt in [
"--den",
]:
den_tag = str(arg)
msg.info(
"plot_ratios.py",
"Config: denominators will be identified using the tag {0}".
format(den_tag),
verbose_level=0)
if opt in ["--show"]:
msg.info("plot_contents_of_yaml.py",
"Config: showing all distributions found",
verbose_level=0)
do_show = True
if opt in ['-s', "--save"]:
save_file = str(arg)
if save_file[-4:] != ".pdf": save_file = save_file + ".pdf"
msg.info("plot_contents_of_yaml.py",
"Config: saving plots to {0}".format(save_file),
verbose_level=0)
if opt in ['-v', "--verbosity"]:
msg.info("plot_ratios.py",
"Config: setting verbosity to {0}".format(arg),
verbose_level=0)
try:
msg.VERBOSE_LEVEL = int(arg)
except:
msg.fatal(
"plot_ratios.py",
"Could not cast verbosity level {0} to integer".format(
arg))
yaml_files = hlp.keep_only_yaml_files(argv_, recurse=do_recurse)
root_files = hlp.keep_only_root_files(argv_, recurse=do_recurse)
if num_tag is None:
num_tag = "measured"
msg.warning(
"plot_ratios.py",
"No --num provided, falling back to \"{0}\"".format(num_tag))
if den_tag is None:
den_tag = "expected"
msg.warning(
"plot_ratios.py",
"No --den provided, falling back to \"{0}\"".format(den_tag))
# Return
return num_tag, den_tag, do_show, save_file, yaml_files, root_files
