def get_uproot_histograms(data_): ret = {} for key, entry in data_.items(): if str(type(entry)) == "<class 'uproot.rootio.TH1F'>": ret[key] = get_ROOT_Table_from_uproot_TH1(entry) elif str(type(entry)) == "<class 'uproot.rootio.TH1D'>": ret[key] = get_ROOT_Table_from_uproot_TH1(entry) elif str(type(entry)) == "<class 'uproot.rootio.TH2F'>": ret[key] = get_ROOT_Table_from_uproot_TH2(entry) elif str(type(entry)) == "<class 'uproot.rootio.TH2D'>": ret[key] = get_ROOT_Table_from_uproot_TH2(entry) elif str(type(entry)) == "<class 'uproot.rootio.TGraph'>": ret[key] = get_ROOT_Table_from_uproot_TGraph(entry) elif str(type(entry)) == "<class 'uproot.rootio.TGraphErrors'>": ret[key] = get_ROOT_Table_from_uproot_TGraphErrors(entry) elif str(type(entry)) == "<class 'uproot.rootio.TGraphAsymmErrors'>": ret[key] = get_ROOT_Table_from_uproot_TGraphAsymmErrors(entry) elif str(type(entry)) == "<class 'uproot.rootio.TGraph2DErrors'>": ret[key] = get_ROOT_Table_from_uproot_TGraph2DErrors(entry) else: msg.info( "HEP_data_utils.ROOT_helpers.get_uproot_histograms", "Object {0} of type {1} is ignored as I don't understand this format" .format(key, type(entry)), verbose_level=0) return ret
def load_yaml_file(dataset_, path_, **kwargs): msg.info("HEP_data_helpers.load_yaml_file", "Opening yaml file {0}".format(path_), verbose_level=0) data = open_yaml_file(path_) if len(data) != 1: msg.error( "HEP_data_helpers.load_yaml_file", "{0} contains {1} tables where I was expecting 1... is the file format correct?" .format(path_, len(data))) return dep_vars = data[0].get("dependent_variables", None) if dep_vars is None: msg.fatal( "HEP_data_helpers.load_yaml_file", "{0} contains no dependent_variables as required".format(path_)) return indep_vars = data[0].get("independent_variables", None) if indep_vars is None: msg.fatal( "HEP_data_helpers.load_yaml_file", "{0} contains no independent_variables as required".format(path_)) return load_distributions_from_yaml(dataset_, dep_vars, indep_vars, path_, **kwargs)
def parse_inputs ( argv_ ) : # Get arguments try : opts, rest = getopt.getopt(argv_,"hrv:",["help","recursive","verbosity=","yaml=","compare=",""]) except getopt.GetoptError as err : msg.error("validate_yaml_files.py","The following error was thrown whilst parsing command-line arguments") print(">>>>>>>>\n",err,"\n<<<<<<<<") msg.error("validate_yaml_files.py","Falling back to to --help...") print_help() msg.fatal("validate_yaml_files.py","Command-line arguments not recognised.") # Parse arguments do_recurse = False for opt, arg in opts: if opt in ['-h',"--help"] : print_help() sys.exit(0) if opt in ['-r',"--recursive",] : msg.info("validate_yaml_files.py","Config: using recursion if needed",verbose_level=0) do_recurse = True if opt in ['-v',"--verbosity"] : msg.info("validate_yaml_files.py","Config: setting verbosity to {0}".format(arg),verbose_level=0) try : msg.VERBOSE_LEVEL = int(arg) except : msg.fatal("validate_yaml_files.py","Could not cast verbosity level {0} to integer".format(arg)) yaml_files = hlp.keep_only_yaml_files(get_argument_list(argv_,"--yaml"),recurse=do_recurse) root_files = hlp.keep_only_root_files(get_argument_list(argv_,"--compare"),recurse=do_recurse) if len(yaml_files) == 0 : msg.fatal("validate_yaml_files.py","Please provide at least one yaml file using the --yaml option") if len(root_files) == 0 : msg.fatal("validate_yaml_files.py","Please provide at least one root file using the --compare option") # Return return yaml_files, root_files
def get_error_from_yaml_map(dep_var_, error_, pt_idx_, err_idx_=0): key = error_.get("label", "err{0}".format(err_idx_)) if "symerror" in error_: if key not in dep_var_._symerrors: msg.info("HEP_data_helpers.get_error_from_yaml_map", "Creating symmetric error {0} with length {1}".format( key, len(dep_var_)), verbose_level=2) dep_var_._symerrors[key] = np.zeros(shape=(len(dep_var_))) dep_var_._symerrors[key][pt_idx_] = float(error_["symerror"]) elif "asymerror" in error_: err_asymm = error_["asymerror"] if key not in dep_var_._asymerrors_up: msg.info("HEP_data_helpers.get_error_from_yaml_map", "Creating asymmetric error {0} with length {1}".format( key, len(dep_var_)), verbose_level=2) dep_var_._asymerrors_up[key] = np.zeros(shape=(len(dep_var_))) dep_var_._asymerrors_dn[key] = np.zeros(shape=(len(dep_var_))) if "plus" not in err_asymm: msg.error("HEP_data_helpers.get_error_from_yaml_map", "No entry named \"plus\" for error \"asymerror\"") else: dep_var_._asymerrors_up[key][pt_idx_] = float(err_asymm["plus"]) if "minus" not in err_asymm: msg.error("HEP_data_helpers.get_error_from_yaml_map", "No entry named \"minus\" for error \"asymerror\"") else: dep_var_._asymerrors_dn[key][pt_idx_] = float(err_asymm["minus"]) else: print(yaml.safe_dump(error_)) msg.error("HEP_data_helpers.get_error_from_yaml_map", "map does not have an entry called symerror or asymerror") return key
def load_root_file(dataset_, path_, **kwargs): msg.info("ROOT_helpers.load_root_file", "Opening root file {0}".format(path_), verbose_level=0) raw_uproot_data = open_root_file(path_, path_) uproot_histos = get_uproot_histograms(raw_uproot_data) for key, table in uproot_histos.items(): load_distribution_from_ROOT_Table(dataset_, key, table, **kwargs)
def set_save_file ( fname_ ) : global document if type(fname_) is str : if type(document) is PdfPages : document.close() if fname_[-4:] != ".pdf" : fname_ = fname_ + ".pdf" msg.info("HEP_data_utils.plotting.set_save_file","Opening pdf file {0}".format(fname_),verbose_level=0) document = PdfPages(fname_) else : msg.error("HEP_data_utils.plotting.set_save_file","Filename must be a str")
def load_root_files_from_list ( dataset_ , dir_ , **kwargs ) : if hlp.is_directory(dir_) : for filename in [ dir_+"/"+f for f in os.listdir(dir_) if is_root_file(f) ] : msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening root file {0}".format(filename),verbose_level=0) load_all_root_files(dataset_,filename,**kwargs) elif type(dir_) == list : for filename in dir_ : if type(filename) != str : continue msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening yaml file {0}".format(filename),verbose_level=0) load_all_root_files(dataset_,filename,**kwargs) else : msg.error("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Input {0} is neither a directory nor a list... returning with nothing done".format(dir_),verbose_level=-1)
def open_yaml_file(path_): yaml_file = open(path_, 'r') data = [] try: for datum in yaml.safe_load_all(yaml_file): msg.info("HEP_data_helpers.open_yaml_file", "yaml file opened with entries:", verbose_level=2) msg.check_verbosity_and_print(yaml.safe_dump(datum), verbose_level=2) data.append(datum) except yaml.YAMLError as exc: print(exc) msg.fatal( "HEP_data_helpers.open_yaml_file", "Exception thrown when opening the yaml file (see previous messages)" ) return data
def rename_key(self, old_key_, new_key_): something_done = False old_key = r"{0}".format(old_key_) new_key = r"{0}".format(new_key_) for key in self._inclusive_distributions: if old_key != key: continue self._inclusive_distributions[ new_key_] = self._inclusive_distributions.pop(old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming inclusive distribution key {1} to {2}". format(self._name, old_key_, new_key_), verbose_level=1) something_done = True for key in self._1D_distributions: if old_key != key: continue self._1D_distributions[new_key_] = self._1D_distributions.pop( old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming 1D distribution key {1} to {2}".format( self._name, old_key_, new_key_), verbose_level=1) something_done = True for key in self._2D_distributions: if old_key != key: continue self._2D_distributions[new_key_] = self._2D_distributions.pop( old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming 2D distribution key {1} to {2}".format( self._name, old_key_, new_key_), verbose_level=1) something_done = True for key in self._ND_distributions: if old_key != key: continue self._ND_distributions[new_key_] = self._ND_distributions.pop( old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming ND distribution key {1} to {2}".format( self._name, old_key_, new_key_), verbose_level=1) something_done = True if not something_done: msg.error( "DistributionContainer.rename_key", "Store \"{0}\" with nothing done for old_key_={1}, new_key_={2}" .format(self._name, old_key_, new_key_), verbose_level=1)
for err in errs_not_match : not_matches.append(err) matches = list(set(matches)) not_matches = list({ x for x in not_matches if x not in matches }) print_match_result ( key_ , central_values_match , matches , not_matches ) # =================================== # # ==== Brief: main program ==== # # =================================== # if __name__ == "__main__" : # # Welcome # msg.info("validate_yaml_files.py","Running program") # # Get input files and settings # yamls_to_load, roots_to_load = parse_inputs(sys.argv[1:]) # # Load input files # yaml_tables = DistributionContainer("yaml_files") HD.load_yaml_files_from_list(yaml_tables,yamls_to_load) # root_tables = DistributionContainer("root_files") RT.load_root_files_from_list(root_tables,roots_to_load) # # Look for matches #
except Exception as e: print(e) msg.error("plot_ratios", "Error when plotting ratio... skipping") print( "=====================================================================================" ) # =================================== # # ==== Brief: main program ==== # # =================================== # if __name__ == "__main__": # # Welcome # msg.info("plot_ratios.py", "Running program") # # Get input files and settings # num_tag, den_tag, do_show, save_file, yamls_to_load, roots_to_load = parse_inputs( sys.argv[1:]) do_save = len(save_file) > 0 if do_save: plotter.set_save_file(save_file) # # Load input files # my_tables = DistributionContainer("my_tables") HD.load_yaml_files_from_list(my_tables, yamls_to_load) RT.load_root_files_from_list(my_tables, roots_to_load) # # Get numerator and denominator tables
def parse_inputs(argv_): # Get arguments try: opts, rest = getopt.getopt(argv_, "hrps:t:v:", [ "help", "recursive", "default-2D-bins", "print", "show", "save=", "type=", "verbosity=" ]) except getopt.GetoptError as err: msg.error( "inspect_yaml.py", "The following error was thrown whilst parsing command-line arguments" ) print(">>>>>>>>\n", err, "\n<<<<<<<<") msg.error("inspect_yaml.py", "Falling back to to --help...") print_help() msg.fatal("inspect_yaml.py", "Command-line arguments not recognised.") # Parse arguments do_recurse = False do_print_all = False do_not_make_matrix = False do_show = False save_file = "" restrict_type = None for opt, arg in opts: if opt in ['-h', "--help"]: print_help() sys.exit(0) if opt in [ '-r', "--recursive", ]: msg.info("inspect_yaml.py", "Config: using recursion if needed", verbose_level=0) do_recurse = True if opt in ["--default-2D-bins"]: msg.info( "inspect_yaml.py", "Config: I will *not* try to convert 2D binning into matrix format", verbose_level=0) do_not_make_matrix = True if opt in ["--print"]: msg.info("inspect_yaml.py", "Config: printing all distributions found", verbose_level=0) do_print_all = True if opt in ["--show"]: msg.info("inspect_yaml.py", "Config: showing all distributions found", verbose_level=0) do_show = True if opt in ['-s', "--save"]: save_file = str(arg) if save_file[-4:] != ".pdf": save_file = save_file + ".pdf" msg.info("inspect_yaml.py", "Config: saving plots to {0}".format(save_file), verbose_level=0) if opt in ['-t', "--type"]: arg = str(arg) if arg not in ["root", "yaml"]: msg.error( "inspect_yaml.py", "{0} option {1} not allowed: allowed inputs are \"root\" or \"yaml\" (deafult is both)" ) else: restrict_type = arg msg.info("inspect_yaml.py", "Config: only reading files of type {0}".format( restrict_type), verbose_level=0) if opt in ['-v', "--verbosity"]: msg.info("inspect_yaml.py", "Config: setting verbosity to {0}".format(arg), verbose_level=0) try: msg.VERBOSE_LEVEL = int(arg) except: msg.fatal( "inspect_yaml.py", "Could not cast verbosity level {0} to integer".format( arg)) # Check that the remaining argument is valid if len(rest) == 0: msg.error("inspect_yaml.py", "No argument provided") print_help() msg.fatal("inspect_yaml.py", "No input yaml file or directory provided") if len(rest) == 1 and hlp.is_directory(rest[0]): msg.info("inspect_yaml.py", "Opening input directory {0}...".format(rest[0]), verbose_level=0) rest = [rest[0] + "/" + f for f in os.listdir(rest[0])] yaml_files = [] if restrict_type == None or restrict_type == "yaml": yaml_files = hlp.keep_only_yaml_files(rest, recurse=do_recurse) root_files = [] if restrict_type == None or restrict_type == "root": root_files = hlp.keep_only_root_files(rest, recurse=do_recurse) if len(yaml_files + root_files) == 0: msg.fatal( "inspect_yaml.py", "No input yaml or root files found from the inputs provided") for f in rest: msg.info("inspect_yaml.py", "Registered input file {0}".format(f), verbose_level=0) # Return return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file
def print_help(): msg.info( "inspect_yaml.py", " Usage: python3 inspect_yaml.py <yaml file(s) OR directory OR submission.yaml file>" ) msg.info( "inspect_yaml.py", " If input files/directory contains a submission.yaml file, all other inputs will be ignored" ) msg.info("inspect_yaml.py", " I assume that you follow the format instructions provided at") msg.info( "inspect_yaml.py", " https://hepdata-submission.readthedocs.io/en/latest/introduction.html" ) msg.info("inspect_yaml.py", " Optional arguments are:") msg.info("inspect_yaml.py", " -h, --help\t\tPrint this help message and close") msg.info( "inspect_yaml.py", " -v, --verbosity\tSet VERBOSE_LEVEL {-1, 0, 1, 2} (-1 by default)" ) msg.info( "inspect_yaml.py", " -r, --recursive\tAllow recursive searching of directories") msg.info( "inspect_yaml.py", " \tRecursion stops if submission.yaml file is found" ) msg.info("inspect_yaml.py", " -s, --save\t\tSave plots to the file provided") msg.info( "inspect_yaml.py", " -t, --type\t\tSpecify input type as { root , yaml } (default is both)" ) msg.info( "inspect_yaml.py", " --print\t\tPrint all information on the distributions found") msg.info("inspect_yaml.py", " --show\t\t\tShow plots to the screen") msg.info( "inspect_yaml.py", " --default-2D-bins\tPrevent interpretation of 2D bins as a matrix" ) msg.info( "inspect_yaml.py", " \t(will be stored as a 1D vector instead)") msg.info( "inspect_yaml.py", " N.B. you can validate your yaml file format using the package:") msg.info("inspect_yaml.py", " https://github.com/HEPData/hepdata-validator")
for f in rest: msg.info("inspect_yaml.py", "Registered input file {0}".format(f), verbose_level=0) # Return return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file # =================================== # # ==== Brief: main program ==== # # =================================== # if __name__ == "__main__": # # Welcome # msg.info("inspect_yaml.py", "Running program") # # Get input files and settings # yamls_to_load, roots_to_load, do_show, do_print_all, do_not_make_matrix, save_file = parse_inputs( sys.argv[1:]) do_save = len(save_file) > 0 if do_save: plotter.set_save_file(save_file) if not do_show and not do_save and not do_print_all: msg.warning( "inspect_yaml.py", "Neither --save, --show nor --print specified. Falling back to --print." ) do_print_all = True # # Load input files
for f in rest: msg.info("plot_contents_of_yaml.py", "Registered input file {0}".format(f), verbose_level=0) # Return return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file # =================================== # # ==== Brief: main program ==== # # =================================== # if __name__ == "__main__": # # Welcome # msg.info("plot_contents_of_yaml.py", "Running program") # # Get input files and settings # yamls_to_load, roots_to_load, do_show, do_print_all, do_not_make_matrix, save_file = parse_inputs( sys.argv[1:]) do_save = len(save_file) > 0 if do_save: plotter.set_save_file(save_file) if not do_show and not do_save and not do_print_all: msg.warning( "plot_contents_of_yaml.py", "Neither --save, --show nor --print specified. Falling back to --print." ) do_print_all = True # # Load input files
def print_help(): msg.info( "plot_ratios.py", "Usage: python3 plot_ratios.py --num <numerator tag> --den <denominator tag> <input-files>" ) msg.info( "plot_ratios.py", "If input files/directory contains a submission.yaml file, all other inputs will be ignored" ) msg.info("plot_ratios.py", "I assume that you follow the format instructions provided at") msg.info( "plot_ratios.py", " https://hepdata-submission.readthedocs.io/en/latest/introduction.html" ) msg.info("plot_ratios.py", "Optional arguments are:") msg.info("plot_ratios.py", " -h, --help\tPrint this help message and close") msg.info( "plot_ratios.py", " -v, --verbosity\tSet VERBOSE_LEVEL {-1, 0, 1, 2} (-1 by default)" ) msg.info("plot_ratios.py", " -r, --recursive\tAllow recursive searching of directories") msg.info( "plot_ratios.py", " \tRecursion stops if submission.yaml file is found" ) msg.info("plot_ratios.py", " -s, --save\t\tSave plots to the file provided") msg.info("plot_ratios.py", " --show\t\tShow plots to the screen") msg.info("plot_ratios.py", "N.B. you can validate your yaml file format using the package:") msg.info("plot_ratios.py", " https://github.com/HEPData/hepdata-validator")
def print_help () : msg.info("validate_yaml_files.py:print_help","Usage is: python3 validate_yaml_files.py --yaml <yaml file(s)> --compare <root file(s)>") msg.info("validate_yaml_files.py:print_help","If input files/directory contains a submission.yaml file, all other inputs will be ignored") msg.info("validate_yaml_files.py:print_help","I assume that you follow the format instructions provided at https://hepdata-submission.readthedocs.io/en/latest/introduction.html") msg.info("validate_yaml_files.py:print_help","Optional arguments are:") msg.info("validate_yaml_files.py:print_help"," -h, --help\t\tPrint this help message and close") msg.info("validate_yaml_files.py:print_help"," -v, --verbosity\t\tSet HEP_data_utils.messaging.VERBOSE_LEVEL as {-1, 0, 1, 2} (-1 by default)") msg.info("validate_yaml_files.py:print_help"," -r, --recursive\t\tAllow recursive searching of directories. Recursion stops if submission.yaml file is found") msg.info("validate_yaml_files.py:print_help","N.B. you can validate your yaml files using the following package: https://github.com/HEPData/hepdata-validator")
print(histo_.xlabel) print(histo_.xvalues) print(histo_.ylabel) print(histo_.yvalues) return print([x for x in dir(histo_) if x[:1] != "_"]) # =================================== # # ==== Brief: main program ==== # # =================================== # if __name__ == "__main__": # # Welcome # msg.info("test_uproot.py", "Running program") # # Get input files and settings # filename = "make_rootfile_output.root" histograms = get_histograms(filename) for key, histo in histograms.items(): print(key) if str(type(histo)) == "<class 'uproot.rootio.TH1'>": read_TH1(histo) elif str(type(histo)) == "<class 'uproot.rootio.TH1F'>": read_TH1(histo) elif str(type(histo)) == "<class 'uproot.rootio.TH1D'>": read_TH1(histo) elif str(type(histo)) == "<class 'uproot.rootio.TH2'>": read_TH2(histo) elif str(type(histo)) == "<class 'uproot.rootio.TH2F'>":
def parse_inputs(argv_): # Get arguments try: opts, rest = getopt.getopt(argv_, "hrv:s:", [ "help", "recursive", "show", "save=", "verbosity=", "num=", "den=" ]) except getopt.GetoptError as err: msg.error( "plot_ratios.py", "The following error was thrown whilst parsing command-line arguments" ) print(">>>>>>>>\n", err, "\n<<<<<<<<") msg.error("plot_ratios.py", "Falling back to to --help...") print_help() msg.fatal("plot_ratios.py", "Command-line arguments not recognised.") # Parse arguments do_recurse = False do_show = False save_file = "" num_tag, den_tag = None, None for opt, arg in opts: if opt in ['-h', "--help"]: print_help() sys.exit(0) if opt in [ '-r', "--recursive", ]: msg.info("plot_ratios.py", "Config: using recursion if needed", verbose_level=0) do_recurse = True if opt in [ "--num", ]: num_tag = str(arg) msg.info("plot_ratios.py", "Config: numerators will be identified using the tag {0}". format(num_tag), verbose_level=0) if opt in [ "--den", ]: den_tag = str(arg) msg.info( "plot_ratios.py", "Config: denominators will be identified using the tag {0}". format(den_tag), verbose_level=0) if opt in ["--show"]: msg.info("plot_contents_of_yaml.py", "Config: showing all distributions found", verbose_level=0) do_show = True if opt in ['-s', "--save"]: save_file = str(arg) if save_file[-4:] != ".pdf": save_file = save_file + ".pdf" msg.info("plot_contents_of_yaml.py", "Config: saving plots to {0}".format(save_file), verbose_level=0) if opt in ['-v', "--verbosity"]: msg.info("plot_ratios.py", "Config: setting verbosity to {0}".format(arg), verbose_level=0) try: msg.VERBOSE_LEVEL = int(arg) except: msg.fatal( "plot_ratios.py", "Could not cast verbosity level {0} to integer".format( arg)) yaml_files = hlp.keep_only_yaml_files(argv_, recurse=do_recurse) root_files = hlp.keep_only_root_files(argv_, recurse=do_recurse) if num_tag is None: num_tag = "measured" msg.warning( "plot_ratios.py", "No --num provided, falling back to \"{0}\"".format(num_tag)) if den_tag is None: den_tag = "expected" msg.warning( "plot_ratios.py", "No --den provided, falling back to \"{0}\"".format(den_tag)) # Return return num_tag, den_tag, do_show, save_file, yaml_files, root_files