def load_all_yaml_files(dataset_, path_, **kwargs): do_recurse = kwargs.get("recurse", True) path_ = hlp.remove_subleading(path_, "/") if hlp.is_directory(path_): yaml_files = hlp.keep_only_yaml_files(path_, recurse=do_recurse) if len(yaml_files) == 0: msg.error( "HEP_data_helpers.load_all_yaml_files", "Directory {0} has no yaml files... returning with nothing done." .format(path_), verbose_level=-1) return for f in yaml_files: load_all_yaml_files(dataset_, f, **kwargs) return if not hlp.is_yaml_file(path_): msg.error( "HEP_data_helpers.load_all_yaml_files", "Path {0} is not a directory or yaml file... returning with nothing done." .format(path_), verbose_level=-1) return if hlp.is_submission_file(path_): load_submission_file(dataset_, path_, **kwargs) return load_yaml_file(dataset_, path_, **kwargs)
def load_yaml_file(dataset_, path_, **kwargs): msg.info("HEP_data_helpers.load_yaml_file", "Opening yaml file {0}".format(path_), verbose_level=0) data = open_yaml_file(path_) if len(data) != 1: msg.error( "HEP_data_helpers.load_yaml_file", "{0} contains {1} tables where I was expecting 1... is the file format correct?" .format(path_, len(data))) return dep_vars = data[0].get("dependent_variables", None) if dep_vars is None: msg.fatal( "HEP_data_helpers.load_yaml_file", "{0} contains no dependent_variables as required".format(path_)) return indep_vars = data[0].get("independent_variables", None) if indep_vars is None: msg.fatal( "HEP_data_helpers.load_yaml_file", "{0} contains no independent_variables as required".format(path_)) return load_distributions_from_yaml(dataset_, dep_vars, indep_vars, path_, **kwargs)
def parse_inputs ( argv_ ) : # Get arguments try : opts, rest = getopt.getopt(argv_,"hrv:",["help","recursive","verbosity=","yaml=","compare=",""]) except getopt.GetoptError as err : msg.error("validate_yaml_files.py","The following error was thrown whilst parsing command-line arguments") print(">>>>>>>>\n",err,"\n<<<<<<<<") msg.error("validate_yaml_files.py","Falling back to to --help...") print_help() msg.fatal("validate_yaml_files.py","Command-line arguments not recognised.") # Parse arguments do_recurse = False for opt, arg in opts: if opt in ['-h',"--help"] : print_help() sys.exit(0) if opt in ['-r',"--recursive",] : msg.info("validate_yaml_files.py","Config: using recursion if needed",verbose_level=0) do_recurse = True if opt in ['-v',"--verbosity"] : msg.info("validate_yaml_files.py","Config: setting verbosity to {0}".format(arg),verbose_level=0) try : msg.VERBOSE_LEVEL = int(arg) except : msg.fatal("validate_yaml_files.py","Could not cast verbosity level {0} to integer".format(arg)) yaml_files = hlp.keep_only_yaml_files(get_argument_list(argv_,"--yaml"),recurse=do_recurse) root_files = hlp.keep_only_root_files(get_argument_list(argv_,"--compare"),recurse=do_recurse) if len(yaml_files) == 0 : msg.fatal("validate_yaml_files.py","Please provide at least one yaml file using the --yaml option") if len(root_files) == 0 : msg.fatal("validate_yaml_files.py","Please provide at least one root file using the --compare option") # Return return yaml_files, root_files
def set_save_file ( fname_ ) : global document if type(fname_) is str : if type(document) is PdfPages : document.close() if fname_[-4:] != ".pdf" : fname_ = fname_ + ".pdf" msg.info("HEP_data_utils.plotting.set_save_file","Opening pdf file {0}".format(fname_),verbose_level=0) document = PdfPages(fname_) else : msg.error("HEP_data_utils.plotting.set_save_file","Filename must be a str")
def asymerrors_dn(self, key_=None): if key_ is None: return self._asymerrors_dn if key_ not in self._asymerrors_dn: msg.error( "DependentVariable.asymerrors_dn", "Key {0} not found in self._asymerrors_dn... returning None.". format(key_), verbose_level=0) return None return self._asymerrors_dn[key_]
def symerrors(self, key_=None): if key_ is None: return self._symerrors if key_ not in self._symerrors: msg.error( "DependentVariable.symerrors", "Key {0} not found in self._symerrors... returning None.". format(key_), verbose_level=0) return None return self._symerrors[key_]
def plot_1D_distribution ( table_ , **kwargs ) : if table_.n_indep_vars() != 1 : msg.error("plotting.plot_1D_distribution","Table has {0} independent variables where 1 was expected".format(table_.n_indep_vars())) return fig = plt.figure(figsize=(10,5)) ax = fig.add_subplot(111) legend_char_width = 53 specific_errors = [] for requested_error in kwargs.get("errors",[]) : if requested_error not in table_._dep_var._symerrors and requested_error not in table_._dep_var._asymerrors_up : msg.warning("plotting.plot_1D_distribution","Specified error {0} not found in dependent variable {1}. Ignoring.".format(requested_error,table_._dep_var._name)) continue specific_errors.append(requested_error) if len(specific_errors) > 0 : x, y, [ex_lo,ex_hi], [ey_lo,ey_hi], labels, keys = get_1D_distribution(table_,specific_errors) ey_lo_sys, ey_hi_sys, ey_lo_stat, ey_hi_stat = [], [], [], [] str_tot_legend = kwargs.get("label","distribution") + " ( " + " + ".join(keys) + " )" str_tot_legend = "\n".join([str_tot_legend[legend_char_width*i:min(len(str_tot_legend),legend_char_width*(i+1))] for i in range(int(len(str_tot_legend)/legend_char_width)+1)]) else : x, y, [ex_lo,ex_hi], [ey_lo,ey_hi], labels, keys = get_1D_distribution(table_) x, y, [ex_lo,ex_hi], [ey_lo_sys,ey_hi_sys], labels, sys_keys = get_1D_distribution(table_,"sys") x, y, [ex_lo,ex_hi], [ey_lo_stat,ey_hi_stat], labels, stat_keys = get_1D_distribution(table_,"stat") str_tot_legend = kwargs.get("label","distribution") + " ( " + " + ".join(keys) + " )" str_tot_legend = "\n".join([str_tot_legend[legend_char_width*i:min(len(str_tot_legend),legend_char_width*(i+1))] for i in range(int(len(str_tot_legend)/legend_char_width)+1)]) str_sys_legend = kwargs.get("label","distribution") + " ( " + " + ".join(sys_keys) + " )" str_sys_legend = "\n".join([str_sys_legend[legend_char_width*i:min(len(str_sys_legend),legend_char_width*(i+1))] for i in range(int(len(str_sys_legend)/legend_char_width)+1)]) if sum([np.fabs(el) for el in ey_hi_sys+ey_lo_sys]) > 0 : ax.errorbar(x, y, yerr=[ey_lo_sys,ey_hi_sys], c='royalblue', linewidth=18, linestyle='None', marker='None', alpha=0.4, label=str_sys_legend) if sum([np.fabs(el) for el in ey_hi_stat+ey_lo_stat]) > 0 : ax.errorbar(x, y, yerr=[ey_lo_stat,ey_hi_stat], c='indianred', linewidth=6, linestyle='None', marker='None', alpha=0.6, label=kwargs.get("label","distribution") + " ( stat )") ax.errorbar(x, y, yerr=[ey_lo,ey_hi], xerr=[ex_lo,ex_hi], c='k', linewidth=2, linestyle='None', marker='+', alpha=1, label=str_tot_legend) if labels : ax.set_xticks(x) ax.set_xticklabels(table_._indep_vars[0]._bin_labels,rotation=90) plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4) if "legend_loc" in kwargs : ax.legend(loc=kwargs.get("legend_loc","best")) else : ax.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) try : plt.xlabel(kwargs.get("xlabel",table_._indep_vars[0].name().replace("\\\\","\\").replace(r"\text{",r"{\rm "))) except : plt.xlabel("<error reading xlabel") try : plt.ylabel(kwargs.get("ylabel",table_._dep_var.name().replace("\\\\","\\").replace(r"\text{",r"{\rm "))) except : plt.ylabel("<error reading ylabel") plt.title(kwargs.get("title","")) xlim = kwargs.get("xlim",[x[0]-np.fabs(ex_lo[0]),x[-1]+np.fabs(ex_hi[-1])]) ylim = kwargs.get("ylim",None) ax.axis(xmin=xlim[0],xmax=xlim[1]) if ylim : ax.axis(ymin=ylim[0],ymax=ylim[1]) if kwargs.get("logy",False) is True : plt.yscale("log") if kwargs.get("logx",False) is True : plt.xscale("log") plt.grid() if kwargs.get("show",False) : plt.show() if kwargs.get("save",False) : fig.savefig ( document , format='pdf' ) plt.close(fig)
def load_root_files_from_list ( dataset_ , dir_ , **kwargs ) : if hlp.is_directory(dir_) : for filename in [ dir_+"/"+f for f in os.listdir(dir_) if is_root_file(f) ] : msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening root file {0}".format(filename),verbose_level=0) load_all_root_files(dataset_,filename,**kwargs) elif type(dir_) == list : for filename in dir_ : if type(filename) != str : continue msg.info("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Opening yaml file {0}".format(filename),verbose_level=0) load_all_root_files(dataset_,filename,**kwargs) else : msg.error("HEP_data_utils.ROOT_helpers.load_root_files_from_list","Input {0} is neither a directory nor a list... returning with nothing done".format(dir_),verbose_level=-1)
def remove_None_entries_from_1D_table(table_): # check dimensions and number of Nones if table_.n_indep_vars() != 1: return dep_var = table_._dep_var indep_var = table_._indep_vars[0] old_values, old_labels, old_centers, old_bw_lo, old_bw_hi = dep_var._values, indep_var._bin_labels, indep_var._bin_centers, indep_var._bin_widths_lo, indep_var._bin_widths_hi num_Nones = len([x for x in old_values if x is None]) if num_Nones == 0: return # make sure bin labels are sensibly configured for a discontinuous distribution if len(old_values) != len(old_labels): msg.error( "HEP_data_utils.HEP_data_helpers.remove_None_entries_from_1D_table", "Cannot interpret binning when len(values) = {0} but len(labels) = {1}" .format(len(old_values), len(old_labels)), verbose_level=1) return for i in range(len(old_labels)): label = old_labels[i] if len(label) > 0: continue old_labels[i] = "[{0},{1}]".format(old_centers[i] - old_bw_lo[i], old_centers[i] + old_bw_hi[i]) # set new values None_indices = [] new_labels = [] new_bw_lo, new_bw_hi = np.full(shape=(len(old_bw_lo) - num_Nones), fill_value=0.5), np.full( shape=(len(old_bw_hi) - num_Nones), fill_value=0.5) new_centers = np.zeros(shape=(len(old_centers) - num_Nones)) for i in range(len(new_centers)): new_centers[i] = i new_bw_lo[i], new_bw_hi[i] = 0.5, 0.5 indep_var._bin_centers, indep_var._bin_widths_lo, indep_var._bin_widths_hi = new_centers, new_bw_lo, new_bw_hi for i in range(len(old_values)): if old_values[i] == None: None_indices.append(i) continue new_labels.append(old_labels[i]) dep_var._values = np.delete(old_values, None_indices) indep_var._bin_labels = new_labels # set new errors for key in dep_var._symerrors: dep_var._symerrors[key] = np.delete(dep_var._symerrors[key], None_indices) for key in dep_var._asymerrors_up: dep_var._asymerrors_up[key] = np.delete(dep_var._asymerrors_up[key], None_indices) for key in dep_var._asymerrors_dn: dep_var._asymerrors_dn[key] = np.delete(dep_var._asymerrors_dn[key], None_indices)
def rename_key(self, old_key_, new_key_): something_done = False old_key = r"{0}".format(old_key_) new_key = r"{0}".format(new_key_) for key in self._inclusive_distributions: if old_key != key: continue self._inclusive_distributions[ new_key_] = self._inclusive_distributions.pop(old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming inclusive distribution key {1} to {2}". format(self._name, old_key_, new_key_), verbose_level=1) something_done = True for key in self._1D_distributions: if old_key != key: continue self._1D_distributions[new_key_] = self._1D_distributions.pop( old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming 1D distribution key {1} to {2}".format( self._name, old_key_, new_key_), verbose_level=1) something_done = True for key in self._2D_distributions: if old_key != key: continue self._2D_distributions[new_key_] = self._2D_distributions.pop( old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming 2D distribution key {1} to {2}".format( self._name, old_key_, new_key_), verbose_level=1) something_done = True for key in self._ND_distributions: if old_key != key: continue self._ND_distributions[new_key_] = self._ND_distributions.pop( old_key_) msg.info( "DistributionContainer.rename_key", "Store \"{0}\" renaming ND distribution key {1} to {2}".format( self._name, old_key_, new_key_), verbose_level=1) something_done = True if not something_done: msg.error( "DistributionContainer.rename_key", "Store \"{0}\" with nothing done for old_key_={1}, new_key_={2}" .format(self._name, old_key_, new_key_), verbose_level=1)
def load_distribution_from_ROOT_Table(dataset_, key_, table_, **kwargs): # Create the table object table = HEPDataTable() table._submission_file_meta = kwargs.get("submission_file_metadata", None) table._submission_file_table = kwargs.get("submission_file_table", None) # Set the variables table._dep_var = ROOT_observable_to_DependentVariable(table_.dep_var) for indep_var in table_.indep_vars: table._indep_vars.append(ROOT_axis_to_IndependentVariable(indep_var)) # Figure out what key to give it if key_ in dataset_: key_ = key_ + "-;1" while key_ in self: key_ = key_[:-1] + str(1 + int(key_[-1:])) # Validate our table is_valid, validity_message = table.is_valid() if not is_valid: msg.error( "ROOT_helpers.load_distribution_from_ROOT_Table", "Error occured when loading table {0}... returning with nothing done." .format(key_)) msg.error("ROOT_helpers.load_distribution_from_ROOT_Table", ">>>>>>>>>>>>>>") msg.error("ROOT_helpers.load_distribution_from_ROOT_Table", validity_message) msg.error("ROOT_helpers.load_distribution_from_ROOT_Table", "<<<<<<<<<<<<<<") return # Add to dataset n_dim = table.n_indep_vars() if n_dim == 0: dataset_._inclusive_distributions[key_] = table elif n_dim == 1: dataset_._1D_distributions[key_] = table elif n_dim == 2: dataset_._2D_distributions[key_] = table else: dataset_._ND_distributions[key_] = table
def plot_2D_distribution_on_current_axes(table_, **kwargs): if table_.n_indep_vars() != 2: msg.error( "HEP_data_utils.plotting.plot_2D_distribution_on_current_canvas", "Table has {0} independent variables where 1 was expected".format( table_.name(), table_.n_indep_vars())) return dep_var, indep_var = table_._dep_var, table_._indep_vars values = dep_var._values ax = plt.gca() try: plt.xlabel( kwargs.get( "x_label", indep_var[0].name().replace("\\\\", "\\").replace(r"\text{", r"{\rm "))) except Exception: pass try: plt.ylabel( kwargs.get( "y_label", indep_var[1].name().replace("\\\\", "\\").replace(r"\text{", r"{\rm "))) except Exception: pass max_val = max([np.fabs(val) for val in values.flatten()]) vmin, vmax = -1 * max_val, max_val if "vlim" in kwargs: vmin = kwargs["vlim"][0] vmax = kwargs["vlim"][1] labels_x, labels_y, values = get_2D_distribution(table_) ax.imshow(values.transpose(), cmap="bwr", vmin=vmin, vmax=vmax) precision = kwargs.get("flt_precision", 2) for i in range(len(labels_x)): for j in range(len(labels_y)): ax.text(i, j, "{0:.{1}f}".format(values[i, j], precision), ha="center", va="center", color="k", fontsize=kwargs.get("fontsize", "xx-small")) ax.set_xticks(np.arange(len(labels_x))) ax.set_xticklabels(labels_x, rotation=90) ax.set_yticks(np.arange(len(labels_y))) ax.set_yticklabels(labels_y)
def plot_ratio ( table_num_ , table_den_ , **kwargs ) : specific_errors_num = [ err for err in kwargs.get("errors",[]) if err in { **table_num_._dep_var._symerrors, **table_num_._dep_var._asymerrors_up } ] specific_errors_den = [ err for err in kwargs.get("errors",[]) if err in { **table_den_._dep_var._symerrors, **table_den_._dep_var._asymerrors_up } ] x_n, y_n, [ex_lo_n,ex_hi_n], [ey_lo_n,ey_hi_n], labels, keys_num = get_1D_distribution(table_num_,specific_errors_num) x_d, y_d, [ex_lo_d,ex_hi_d], [ey_lo_d,ey_hi_d], labels, keys_den = get_1D_distribution(table_den_,specific_errors_den) ex_lo_d, ex_hi_d = np.zeros(shape=(len(x_d))), np.zeros(shape=(len(x_d))) for i in range(len(x_n)) : if x_n[i] == x_d[i] : continue msg.error("HEP_data_helpers.plot_ratio","Arguments do not have the same binning") raise ValueError("Ratio of distributions with bin centres at {0} and {1}",x_n.all(),x_d.all()) fig = plt.figure(figsize=(10,10)) ax1 = fig.add_subplot(211) legend_char_width = 53 str_num_legend = kwargs.get("numerator_label","numerator") + " ( " + " + ".join(keys_num) + " )" str_num_legend = "\n".join([str_num_legend[legend_char_width*i:min(len(str_num_legend),legend_char_width*(i+1))] for i in range(int(len(str_num_legend)/legend_char_width)+1)]) str_den_legend = kwargs.get("denominator_label","denominator") + " ( " + " + ".join(keys_den) + " )" str_den_legend = "\n".join([str_den_legend[legend_char_width*i:min(len(str_den_legend),legend_char_width*(i+1))] for i in range(int(len(str_den_legend)/legend_char_width)+1)]) ax1.errorbar(x_d, y_d, yerr=[ey_lo_d,ey_hi_d], xerr=[ex_lo_d,ex_hi_d], c='r', linewidth=7, linestyle='None', marker='+', alpha=0.5, label=str_den_legend) ax1.errorbar(x_n, y_n, yerr=[ey_lo_n,ey_hi_n], xerr=[ex_lo_n,ex_hi_n], c='k', linestyle='None', alpha=0.5, label=str_num_legend) if "legend_loc" in kwargs : ax1.legend(loc=kwargs.get("legend_loc","best")) else : ax1.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4) try : plt.ylabel(kwargs.get("ylabel",table_num_._dep_var.name().replace("\\\\","\\").replace(r"\text{",r"{\rm "))) except : plt.ylabel("<error reading ylabel") plt.title(kwargs.get("title","")) xlim = kwargs.get("xlim",[x_d[0]-np.fabs(ex_lo_n[0]),x_d[-1]+np.fabs(ex_hi_n[-1])]) ylim = kwargs.get("ylim",None) ax1.axis(xmin=xlim[0],xmax=xlim[1]) if ylim : ax1.axis(ymin=ylim[0],ymax=ylim[1]) if kwargs.get("logy",False) is True : plt.yscale("log") if kwargs.get("logx",False) is True : plt.xscale("log") plt.grid() ax2 = fig.add_subplot(212) ax2.errorbar(x_d, y_d/y_d, yerr=[ey_lo_d/y_d,ey_hi_d/y_d], xerr=[ex_lo_d,ex_hi_d], c='r', linewidth=7, linestyle='None', marker='+', alpha=0.5) ax2.errorbar(x_n, y_n/y_d, yerr=[ey_lo_n/y_d,ey_hi_n/y_d], xerr=[ex_lo_n,ex_hi_n], c='k', linestyle='None', alpha=0.5) box = ax2.get_position() ax2.set_position([box.x0, box.y0, box.width * 0.4, box.height]) ax2.axis(xmin=xlim[0],xmax=xlim[1]) plt.ylabel("Ratio") try : plt.xlabel(kwargs.get("xlabel",table_den_._indep_vars[0].name().replace("\\\\","\\").replace(r"\text{",r"{\rm "))) except : plt.xlabel("<error reading xlabel") plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4) plt.grid() if kwargs.get("show",False) : plt.show() if kwargs.get("save",False) : fig.savefig ( document , format='pdf' ) plt.close(fig)
def load_submission_file(dataset_, path_, **kwargs): data = open_yaml_file(path_) # First get the dataset metadata sub_file_meta = SubmissionFileMeta() indices_of_metadata = [] for idx in range(len(data)): dataset_properties = data[idx] if "data_file" in dataset_properties: continue indices_of_metadata.append(idx) additional_resources = dataset_properties.get("additional_resources", []) for idx in range(len(additional_resources)): sub_file_meta._additional_resources.append( additional_resources[idx]) if "comment" in dataset_properties: sub_file_meta._comment = dataset_properties["comment"] if "hepdata_doi" in dataset_properties: sub_file_meta._hepdata_doi = dataset_properties["hepdata_doi"] # Now load tables from all remaining entries for idx in range(len(data)): if idx in indices_of_metadata: continue table = data[idx] data_file = hlp.get_directory(path_) + "/" + table["data_file"] if not os.path.isfile(data_file): msg.error( "HEP_data_helpers.load_submission_file", "Submission file asks for a yaml file called {0} but none exists" .format(data_file)) return sub_file_table = SubmissionFileTable() sub_file_table._data_file = data_file sub_file_table._name = table.get("name", "") sub_file_table._table_doi = table.get("table_doi", "") sub_file_table._location = table.get("location", "") sub_file_table._description = table.get("description", "") sub_file_table._keywords = [(d.get("name", ""), d.get("values", "")) for d in table.get("keywords", [])] load_yaml_file(dataset_, data_file, submission_file_metadata=sub_file_meta, submission_file_table=sub_file_table, **kwargs)
def load_keys(self, filename_): config = configparser.ConfigParser() config.optionxform = str try: config.read(filename_) except Exception as exc: msg.check_verbosity_and_print(exc, verbose_level=-1) msg.error( "HEP_data_utils.data_structures.DistributionContainer", "An exception occured when parsing the config file... Continuing with nothing done" ) return if "KEYS" not in config.sections(): msg.error( "HEP_data_utils.data_structures.DistributionContainer", "No section titled \"KEYS\" in file {0}".format(filename_)) return keys = config["KEYS"] for old_key in keys: self.rename_key(old_key, keys[old_key]) self.print_keys()
def get_error_from_yaml_map(dep_var_, error_, pt_idx_, err_idx_=0): key = error_.get("label", "err{0}".format(err_idx_)) if "symerror" in error_: if key not in dep_var_._symerrors: msg.info("HEP_data_helpers.get_error_from_yaml_map", "Creating symmetric error {0} with length {1}".format( key, len(dep_var_)), verbose_level=2) dep_var_._symerrors[key] = np.zeros(shape=(len(dep_var_))) dep_var_._symerrors[key][pt_idx_] = float(error_["symerror"]) elif "asymerror" in error_: err_asymm = error_["asymerror"] if key not in dep_var_._asymerrors_up: msg.info("HEP_data_helpers.get_error_from_yaml_map", "Creating asymmetric error {0} with length {1}".format( key, len(dep_var_)), verbose_level=2) dep_var_._asymerrors_up[key] = np.zeros(shape=(len(dep_var_))) dep_var_._asymerrors_dn[key] = np.zeros(shape=(len(dep_var_))) if "plus" not in err_asymm: msg.error("HEP_data_helpers.get_error_from_yaml_map", "No entry named \"plus\" for error \"asymerror\"") else: dep_var_._asymerrors_up[key][pt_idx_] = float(err_asymm["plus"]) if "minus" not in err_asymm: msg.error("HEP_data_helpers.get_error_from_yaml_map", "No entry named \"minus\" for error \"asymerror\"") else: dep_var_._asymerrors_dn[key][pt_idx_] = float(err_asymm["minus"]) else: print(yaml.safe_dump(error_)) msg.error("HEP_data_helpers.get_error_from_yaml_map", "map does not have an entry called symerror or asymerror") return key
def plot(self, key_, **kwargs): for key, dist in self._inclusive_distributions.items(): if key != key_: continue try: plotter.plot_inclusive_distribution(dist, **kwargs) except Exception as e: print(e) msg.error( "HEP_data_utils.data_structures.DistributionContainer.plot", "Error when plotting inclusive distribution with key {0}... skipping" .format(key)) for key, dist in self._1D_distributions.items(): if key != key_: continue try: plotter.plot_1D_distribution(dist, **kwargs) except Exception as e: print(e) msg.error( "HEP_data_utils.data_structures.DistributionContainer.plot", "Error when plotting 1D distribution with key {0}... skipping" .format(key)) for key, dist in self._2D_distributions.items(): if key != key_: continue try: plotter.plot_2D_distribution(dist, **kwargs) except Exception as e: print(e) msg.error( "HEP_data_utils.data_structures.DistributionContainer.plot", "Error when plotting 2D distribution with key {0}... skipping" .format(key))
def load_distribution_from_yaml(dataset_, dep_var_, indep_vars_, path_, **kwargs): # Create the table object table = HEPDataTable() table._submission_file_meta = kwargs.get("submission_file_metadata", None) table._submission_file_table = kwargs.get("submission_file_table", None) # Set the dependent variable dep_var = get_dependent_variable_from_yaml_map(dep_var_, path_) table._dep_var = dep_var # Set the independent variables for indep_var_map in indep_vars_: indep_var = IndependentVariable() indep_header = indep_var_map.get("header", None) if indep_header: indep_var._name = indep_header.get("name", "") indep_var._units = indep_header.get("units", "") indep_var._bin_centers, indep_var._bin_widths_lo, indep_var._bin_widths_hi, indep_var._bin_labels = get_bins_from_dict( indep_var_map, len(dep_var)) table._indep_vars.append(indep_var) # If our table has 'None' entries, we want to remove them remove_None_entries_from_1D_table(table) # Figure out what key to give it key = dataset_.generate_key(table) # Validate our table is_valid, validity_message = table.is_valid() if not is_valid: msg.error( "HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml", "Error occured when loading table {0} from file {1}... returning with nothing done." .format(key, path_)) msg.error( "HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml", ">>>>>>>>>>>>>>") msg.error( "HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml", validity_message) msg.error( "HEP_data_utils.HEP_data_helpers.load_distribution_from_yaml", "<<<<<<<<<<<<<<") return # Reformat 2D distribution bins into a matrix n_dim = len(table._indep_vars) if n_dim == 2 and dataset_._make_matrix_if_possible: reformat_2D_bins_as_matrix(table) # Add to dataset if n_dim == 0: dataset_._inclusive_distributions[key] = table elif n_dim == 1: dataset_._1D_distributions[key] = table elif n_dim == 2: dataset_._2D_distributions[key] = table else: dataset_._ND_distributions[key] = table
def plot_2D_distribution(table_, **kwargs): if table_.n_indep_vars() != 2: msg.error( "HEP_data_utils.plotting.plot_2D_distribution", "Table has {0} independent variables where 1 was expected".format( table_.name(), table_.n_indep_vars())) return dep_var, indep_var = table_._dep_var, table_._indep_vars values = dep_var._values fig = plt.figure(figsize=(7, 7)) ax = fig.add_subplot(111) plot_2D_distribution_on_current_axes(table_, **kwargs) try: plt.title(table_._dep_var.name().replace("\\\\", "\\").replace( r"\text{", r"{\rm ")) except Exception: pass if kwargs.get("show", False): plt.show() if kwargs.get("save", False): fig.savefig(document, format='pdf') plt.close(fig)
def plot_2D_distribution ( table_ , **kwargs ) : if table_.n_indep_vars() != 2 : msg.error("plotting.plot_2D_distribution","Table has {0} independent variables where 1 was expected".format(table_.name(),table_.n_indep_vars())) return dep_var = table_._dep_var indep_var = table_._indep_vars values = dep_var._values fig = plt.figure(figsize=(7,7)) ax = fig.add_subplot(111) #x_label = str([ "{0} [{1}:{2}]".format(var,table_._local_key_indices[var][0],table_._local_key_indices[var][1]) for var in table_._local_keys ]) try : x_label = kwargs.get("x_label",indep_var[0].name().replace("\\\\","\\").replace(r"\text{",r"{\rm ")) except : plt.xlabel("<error reading xlabel") try : y_label = kwargs.get("y_label",indep_var[1].name().replace("\\\\","\\").replace(r"\text{",r"{\rm ")) except : plt.ylabel("<error reading ylabel") max_val = max([np.fabs(val) for val in values.flatten()]) vmin = -1*max_val vmax = max_val if "vlim" in kwargs : vmin = kwargs["vlim"][0] vmax = kwargs["vlim"][1] labels_x, labels_y, values = get_2D_distribution(table_) ax.imshow(values,cmap="bwr",vmin=vmin,vmax=vmax) plt.xlabel(kwargs.get("xlabel",x_label)) plt.ylabel(kwargs.get("ylabel",y_label)) precision = kwargs.get("flt_precision",2) for i in range(len(labels_x)) : for j in range(len(labels_y)) : ax.text(j, i, "{0:.{1}f}".format(values[i, j],precision), ha="center", va="center", color="k",fontsize="xx-small") ax.set_xticks(np.arange(len(labels_x))) ax.set_xticklabels(labels_x,rotation=90) ax.set_yticks(np.arange(len(labels_y))) ax.set_yticklabels(labels_y) plt.title(kwargs.get("title",dep_var.name().replace("\\\\","\\").replace(r"\text{",r"{\rm "))) if kwargs.get("show",False) : plt.show() if kwargs.get("save",False) : fig.savefig ( document , format='pdf' ) plt.close(fig)
def plot_and_print(num_dist, den_dist, **kwargs): if not HD.has_matching_bins(num_dist, den_dist): return n_dim = num_dist.n_indep_vars() print( "=====================================================================================" ) print("=== PLOTTING THE FOLLOWING RATIO ( 1 divided by 2 )") print("=== 1. ", num_dist._dep_var._name) print("=== 2. ", den_dist._dep_var._name) if n_dim == 1: print("=== as a function of: ", num_dist._indep_vars[0]._name) if n_dim == 2: print("=== as a function of: ", num_dist._indep_vars[0]._name) print("=== ", num_dist._indep_vars[1]._name) try: chi2, bin_ratios = plot_ratio(num_dist, den_dist, **kwargs) print("=== chi2 = {:.4f}".format(chi2)) print("=== bin ratios = ", ["{:.4f}".format(x) for x in bin_ratios]) except Exception as e: print(e) msg.error("plot_ratios", "Error when plotting ratio... skipping") print( "=====================================================================================" )
def get_dependent_variable_from_yaml_map(dep_var_, path_="unknown"): dep_var = DependentVariable() dep_header = dep_var_.get("header", None) if dep_header: dep_var._name = dep_header.get("name", "") dep_var._units = dep_header.get("units", "") dep_qualifiers = dep_var_.get("qualifiers", None) if dep_qualifiers: if type(dep_qualifiers) is not list: msg.error( "HEP_data_helpers.get_dependent_variable_from_yaml_map", "Dependent variable {0} in file {1} has qualifiers stored in type {2} where I was expecting a list" .format(dep_var._name, path_, type(dep_qualifiers)), verbose_level=0) else: for entry in dep_qualifiers: if type(entry) is dict: name, value = entry.get("name", None), str(entry.get("value", "")) if name: dep_var._qualifiers.append((name, value)) for entry in dep_var_.get("values", []): try: val = float(entry["value"]) except ValueError as exc: msg.error( "HEP_data_helpers.get_dependent_variable_from_yaml_map", "ValueError ({0}) when loading dependent variable {1} from file {2}" .format(exc, dep_var._name, path_), verbose_level=1) val = None except KeyError as exc: msg.error( "HEP_data_helpers.get_dependent_variable_from_yaml_map", "KeyError ({0}) when loading dependent variable {1} from file {2}" .format(exc, dep_var._name, path_), verbose_level=1) val = None dep_var._values = np.append(dep_var._values, val) pt_idx = 0 for entry in dep_var_.get("values", []): errors = entry.get("errors", []) err_idx = 0 for error in errors: get_error_from_yaml_map(dep_var, error, pt_idx, err_idx) err_idx = err_idx + 1 pt_idx = pt_idx + 1 return dep_var
def plot_ratio(self, key_num_, key_den_, **kwargs): table_num = self._1D_distributions.get(key_num_, None) if not table_num: msg.error( "HEP_data_utils.data_structures.DistributionContainer.plot_ratio", "Error when plotting 1D distribution with key {0}... skipping". format(key_num_)) raise KeyError("key {0} not in {1}".format(key_num_, self._name)) table_den = self._1D_distributions.get(key_den_, None) if not table_den: msg.error( "HEP_data_utils.data_structures.DistributionContainer.plot_ratio", "Error when plotting 1D distribution with key {0}... skipping". format(key_den_)) raise KeyError("key {0} not in {1}".format(key_den_, self._name)) try: plotter.plot_ratio(table_num, table_den, **kwargs) except Exception as e: print(e) msg.error( "HEP_data_utils.data_structures.DistributionContainer.plot_ratio", "Error when plotting {0} / {1} ratio... skipping".format( key_num_, key_den_))
def parse_inputs(argv_): # Get arguments try: opts, rest = getopt.getopt(argv_, "hrps:t:v:", [ "help", "recursive", "default-2D-bins", "print", "show", "save=", "type=", "verbosity=" ]) except getopt.GetoptError as err: msg.error( "inspect_yaml.py", "The following error was thrown whilst parsing command-line arguments" ) print(">>>>>>>>\n", err, "\n<<<<<<<<") msg.error("inspect_yaml.py", "Falling back to to --help...") print_help() msg.fatal("inspect_yaml.py", "Command-line arguments not recognised.") # Parse arguments do_recurse = False do_print_all = False do_not_make_matrix = False do_show = False save_file = "" restrict_type = None for opt, arg in opts: if opt in ['-h', "--help"]: print_help() sys.exit(0) if opt in [ '-r', "--recursive", ]: msg.info("inspect_yaml.py", "Config: using recursion if needed", verbose_level=0) do_recurse = True if opt in ["--default-2D-bins"]: msg.info( "inspect_yaml.py", "Config: I will *not* try to convert 2D binning into matrix format", verbose_level=0) do_not_make_matrix = True if opt in ["--print"]: msg.info("inspect_yaml.py", "Config: printing all distributions found", verbose_level=0) do_print_all = True if opt in ["--show"]: msg.info("inspect_yaml.py", "Config: showing all distributions found", verbose_level=0) do_show = True if opt in ['-s', "--save"]: save_file = str(arg) if save_file[-4:] != ".pdf": save_file = save_file + ".pdf" msg.info("inspect_yaml.py", "Config: saving plots to {0}".format(save_file), verbose_level=0) if opt in ['-t', "--type"]: arg = str(arg) if arg not in ["root", "yaml"]: msg.error( "inspect_yaml.py", "{0} option {1} not allowed: allowed inputs are \"root\" or \"yaml\" (deafult is both)" ) else: restrict_type = arg msg.info("inspect_yaml.py", "Config: only reading files of type {0}".format( restrict_type), verbose_level=0) if opt in ['-v', "--verbosity"]: msg.info("inspect_yaml.py", "Config: setting verbosity to {0}".format(arg), verbose_level=0) try: msg.VERBOSE_LEVEL = int(arg) except: msg.fatal( "inspect_yaml.py", "Could not cast verbosity level {0} to integer".format( arg)) # Check that the remaining argument is valid if len(rest) == 0: msg.error("inspect_yaml.py", "No argument provided") print_help() msg.fatal("inspect_yaml.py", "No input yaml file or directory provided") if len(rest) == 1 and hlp.is_directory(rest[0]): msg.info("inspect_yaml.py", "Opening input directory {0}...".format(rest[0]), verbose_level=0) rest = [rest[0] + "/" + f for f in os.listdir(rest[0])] yaml_files = [] if restrict_type == None or restrict_type == "yaml": yaml_files = hlp.keep_only_yaml_files(rest, recurse=do_recurse) root_files = [] if restrict_type == None or restrict_type == "root": root_files = hlp.keep_only_root_files(rest, recurse=do_recurse) if len(yaml_files + root_files) == 0: msg.fatal( "inspect_yaml.py", "No input yaml or root files found from the inputs provided") for f in rest: msg.info("inspect_yaml.py", "Registered input file {0}".format(f), verbose_level=0) # Return return yaml_files, root_files, do_show, do_print_all, do_not_make_matrix, save_file
def plot_ratio_1D(table_num_, table_den_, **kwargs): x_n, y_n, [ex_lo_n, ex_hi_n], [ ey_lo_n, ey_hi_n ], labels, keys_num = plotter.get_1D_distribution(table_num_) x_d, y_d, [ex_lo_d, ex_hi_d], [ ey_lo_d, ey_hi_d ], labels, keys_den = plotter.get_1D_distribution(table_den_) chi2 = hlp.get_chi2(y_n, ey_lo_n, ey_hi_n, y_d, ey_lo_d, ey_hi_d) for i in range(len(x_n)): if x_n[i] == x_d[i]: continue msg.error("plot_ratio_1D", "Arguments do not have the same binning") raise ValueError( "Ratio of distributions with bin centres at {0} and {1}", x_n, x_d) fig = plotter.plt.figure(figsize=(10, 10)) ax1 = fig.add_subplot(211) legend_char_width = 53 str_num_legend = "TABLE 1 ( " + " + ".join(keys_num) + " )" str_num_legend = "\n".join([ str_num_legend[legend_char_width * i:min(len(str_num_legend), legend_char_width * (i + 1))] for i in range(int(len(str_num_legend) / legend_char_width) + 1) ]) str_den_legend = "TABLE 2 ( " + " + ".join(keys_den) + " )" str_den_legend = "\n".join([ str_den_legend[legend_char_width * i:min(len(str_den_legend), legend_char_width * (i + 1))] for i in range(int(len(str_den_legend) / legend_char_width) + 1) ]) ax1.errorbar(x_n, y_n, yerr=[ey_lo_n, ey_hi_n], xerr=[ex_lo_n, ex_hi_n], c='k', linewidth=2, linestyle='None', alpha=0.8, label=str_num_legend) ax1.errorbar(x_d, y_d, yerr=[ey_lo_d, ey_hi_d], xerr=[ex_lo_d, ex_hi_d], c='r', linewidth=4, linestyle='None', alpha=0.4, label=str_den_legend) ax1.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) plotter.plt.subplots_adjust(left=0.1, right=0.5, top=0.95, bottom=0.4) plotter.plt.ylabel("Values") plotter.plt.ylabel("Values") xlim = kwargs.get( "xlim", [x_d[0] - np.fabs(ex_lo_n[0]), x_d[-1] + np.fabs(ex_hi_n[-1])]) ymin, ymax = ax1.get_ylim() ax1.axis(xmin=xlim[0], xmax=xlim[1]) try: plotter.plt.text( xlim[0], 1.19 * ymax - 0.19 * ymin, ("$\\bf{TABLE}$ $\\bf{1:}$ " + table_num_._dep_var._name).replace( "\\\\", "\\").replace(r"\text{", r"{\rm ")) plotter.plt.text( xlim[0], 1.08 * ymax - 0.08 * ymin, ("$\\bf{TABLE}$ $\\bf{2:}$ " + table_den_._dep_var._name).replace( "\\\\", "\\").replace(r"\text{", r"{\rm ")) except: msg.warning( "plot_ratio_1D", "could not render observable name - no title given to plot") plotter.plt.grid() ax2 = fig.add_subplot(212) ax2.errorbar(x_n, y_n / y_d, yerr=[ey_lo_n / y_d, ey_hi_n / y_d], xerr=[ex_lo_n, ex_hi_n], c='k', linewidth=2, linestyle='None', alpha=0.8) ax2.errorbar(x_d, y_d / y_d, yerr=[ey_lo_d / y_d, ey_hi_d / y_d], xerr=[ex_lo_d, ex_hi_d], c='r', linewidth=4, linestyle='None', alpha=0.4) box = ax2.get_position() ax2.set_position([box.x0, box.y0, box.width * 0.4, box.height]) ax2.axis(xmin=xlim[0], xmax=xlim[1]) plotter.plt.ylabel("Ratio $\\bf{vs.}$ TABLE 2") try: plotter.plt.xlabel( kwargs.get( "xlabel", table_den_._indep_vars[0].name().replace( "\\\\", "\\").replace(r"\text{", r"{\rm "))) except: plotter.plt.xlabel("<error reading xlabel") plotter.plt.subplots_adjust(left=0.1, right=0.5, top=0.92, bottom=0.4) plotter.plt.grid() if kwargs.get("show", False): plotter.plt.show() if kwargs.get("save", False): fig.savefig(plotter.document, format='pdf') plotter.plt.close(fig) return chi2, y_n / y_d
def parse_inputs(argv_): # Get arguments try: opts, rest = getopt.getopt(argv_, "hrv:s:", [ "help", "recursive", "show", "save=", "verbosity=", "num=", "den=" ]) except getopt.GetoptError as err: msg.error( "plot_ratios.py", "The following error was thrown whilst parsing command-line arguments" ) print(">>>>>>>>\n", err, "\n<<<<<<<<") msg.error("plot_ratios.py", "Falling back to to --help...") print_help() msg.fatal("plot_ratios.py", "Command-line arguments not recognised.") # Parse arguments do_recurse = False do_show = False save_file = "" num_tag, den_tag = None, None for opt, arg in opts: if opt in ['-h', "--help"]: print_help() sys.exit(0) if opt in [ '-r', "--recursive", ]: msg.info("plot_ratios.py", "Config: using recursion if needed", verbose_level=0) do_recurse = True if opt in [ "--num", ]: num_tag = str(arg) msg.info("plot_ratios.py", "Config: numerators will be identified using the tag {0}". format(num_tag), verbose_level=0) if opt in [ "--den", ]: den_tag = str(arg) msg.info( "plot_ratios.py", "Config: denominators will be identified using the tag {0}". format(den_tag), verbose_level=0) if opt in ["--show"]: msg.info("plot_contents_of_yaml.py", "Config: showing all distributions found", verbose_level=0) do_show = True if opt in ['-s', "--save"]: save_file = str(arg) if save_file[-4:] != ".pdf": save_file = save_file + ".pdf" msg.info("plot_contents_of_yaml.py", "Config: saving plots to {0}".format(save_file), verbose_level=0) if opt in ['-v', "--verbosity"]: msg.info("plot_ratios.py", "Config: setting verbosity to {0}".format(arg), verbose_level=0) try: msg.VERBOSE_LEVEL = int(arg) except: msg.fatal( "plot_ratios.py", "Could not cast verbosity level {0} to integer".format( arg)) yaml_files = hlp.keep_only_yaml_files(argv_, recurse=do_recurse) root_files = hlp.keep_only_root_files(argv_, recurse=do_recurse) if num_tag is None: num_tag = "measured" msg.warning( "plot_ratios.py", "No --num provided, falling back to \"{0}\"".format(num_tag)) if den_tag is None: den_tag = "expected" msg.warning( "plot_ratios.py", "No --den provided, falling back to \"{0}\"".format(den_tag)) # Return return num_tag, den_tag, do_show, save_file, yaml_files, root_files
) print("=== PLOTTING THE FOLLOWING RATIO ( 1 divided by 2 )") print("=== 1. ", num_dist._dep_var._name) print("=== 2. ", den_dist._dep_var._name) try: chi2, bin_ratios = plot_ratio(num_dist, den_dist, show=do_show, save=do_save) print("=== chi2 = {:.4f}".format(chi2)) print("=== bin ratios = ", ["{:.4f}".format(x) for x in bin_ratios]) except Exception as e: print(e) msg.error( "HEP_data_utils.data_structures.DistributionContainer.plot_ratio", "Error when plotting ratio... skipping") print( "=====================================================================================" ) # # Close save file # if do_save: plotter.close_save_file() # # Goodbye # msg.info( "plot_ratio_between_distributions.py", "Program reached the end without crashing and will close :) Have a nice day..." )