def xsecs( self, thetas=None, nus=None, partition="all", test_split=0.2, validation_split=0.2, include_nuisance_benchmarks=True, batch_size=100000, generated_close_to=None, ): """ Returns the total cross sections for benchmarks or parameter points. Parameters ---------- thetas : None or list of (ndarray or str), optional If None, the function returns all benchmark cross sections. Otherwise, it returns the cross sections for a series of parameter points that are either given by their benchmark name (as a str), their benchmark index (as an int), or their parameter value (as an ndarray, using morphing). Default value: None. nus : None or list of (None or ndarray), optional If None, the nuisance parameters are set to their nominal values (0), i.e. no systematics are taken into account. Otherwise, the list has to have the same number of elements as thetas, and each entry can specify nuisance parameters at nominal value (None) or a value of the nuisance parameters (ndarray). partition : {"train", "test", "validation", "all"}, optional Which event partition to use. Default: "all". test_split : float, optional Fraction of events reserved for testing. Default value: 0.2. validation_split : float, optional Fraction of weighted events reserved for validation. Default value: 0.2. include_nuisance_benchmarks : bool, optional Whether to include nuisance benchmarks if thetas is None. Default value: True. batch_size : int, optional Size of the batches of events that are loaded into memory at the same time. Default value: 100000. generated_close_to : None or ndarray, optional If not None, only events originally generated from the closest benchmark to this parameter point will be used. Default value : None. Returns ------- xsecs : ndarray Calculated cross sections in pb. xsec_uncertainties : ndarray Cross-section uncertainties in pb. Basically calculated as sum(weights**2)**0.5. """ logger.debug("Calculating cross sections for thetas = %s and nus = %s", thetas, nus) # Inputs if thetas is not None: include_nuisance_benchmarks = True if thetas is not None: if nus is None: nus = [None for _ in thetas] assert len(nus) == len(thetas), "Numbers of thetas and nus don't match!" # Which events to use if partition == "all": start_event, end_event = None, None correction_factor = 1.0 elif partition in ["train", "validation", "test"]: start_event, end_event, correction_factor = self._train_validation_test_split( partition, test_split, validation_split ) else: raise ValueError(f"Invalid partition type: {partition}") # Theta matrices (translation of benchmarks to theta, at nominal nuisance params) if thetas is None: theta_matrices = np.identity(self.n_benchmarks) else: theta_matrices = [self._get_theta_benchmark_matrix(theta) for theta in thetas] theta_matrices = np.asarray(theta_matrices) # Shape (n_thetas, n_benchmarks) # Loop over events xsecs = 0.0 xsec_uncertainties = 0.0 n_events = 0 for i_batch, (_, benchmark_weights) in enumerate( self.event_loader( start=start_event, end=end_event, include_nuisance_parameters=include_nuisance_benchmarks, batch_size=batch_size, generated_close_to=generated_close_to, ) ): n_batch, _ = benchmark_weights.shape n_events += n_batch # Benchmark xsecs if thetas is None: xsecs += np.sum(benchmark_weights, axis=0) xsec_uncertainties += np.sum(benchmark_weights * benchmark_weights, axis=0) # xsecs at given parameters(theta, nu) else: # Weights at nominal nuisance params (nu=0) weights_nom = mdot(theta_matrices, benchmark_weights) # Shape (n_thetas, n_batch) weights_sq_nom = mdot(theta_matrices, benchmark_weights * benchmark_weights) # same # Effect of nuisance parameters nuisance_factors = self._calculate_nuisance_factors(nus, benchmark_weights) weights = nuisance_factors * weights_nom weights_sq = nuisance_factors * weights_sq_nom # Sum up xsecs += np.sum(weights, axis=1) xsec_uncertainties += np.sum(weights_sq, axis=1) if n_events == 0: raise RuntimeError( "Did not find events with test_split = %s and generated_close_to = %s", test_split, generated_close_to ) xsec_uncertainties = np.maximum(xsec_uncertainties, 0.0) ** 0.5 # Correct for not using all events xsecs *= correction_factor xsec_uncertainties *= correction_factor logger.debug("xsecs and uncertainties [pb]:") for this_xsec, this_uncertainty in zip(xsecs, xsec_uncertainties): logger.debug(" (%4f +/- %4f) pb (%4f %%)", this_xsec, this_uncertainty, 100 * this_uncertainty / this_xsec) return xsecs, xsec_uncertainties
def _weight_gradients( self, thetas, nus, benchmark_weights, gradients="all", theta_matrices=None, theta_gradient_matrices=None ): """ Turns benchmark weights into weights for given parameter points (theta, nu). Parameters ---------- thetas : list of (ndarray or str) If None, the function returns all benchmark cross sections. Otherwise, it returns the cross sections for a series of parameter points that are either given by their benchmark name (as a str), their benchmark index (as an int), or their parameter value (as an ndarray, using morphing). nus : None or list of (None or ndarray) If None, the nuisance parameters are set to their nominal values (0), i.e. no systematics are taken into account. Otherwise, the list has to have the same number of elements as thetas, and each entry can specify nuisance parameters at nominal value (None) or a value of the nuisance parameters (ndarray). gradients : {"all", "theta", "nu"}, optional Which gradients to calculate. Default value: "all". Returns ------- gradients : ndarray Calculated gradients in pb. """ n_events, _ = benchmark_weights.shape # Inputs if gradients == "all" and self.n_nuisance_parameters == 0: gradients = "theta" if nus is None: nus = [None for _ in thetas] assert len(nus) == len(thetas), "Numbers of thetas and nus don't match!" # Theta matrices (translation of benchmarks to theta, at nominal nuisance params) if theta_matrices is None: theta_matrices = [self._get_theta_benchmark_matrix(theta) for theta in thetas] if theta_gradient_matrices is None: theta_gradient_matrices = [self._get_dtheta_benchmark_matrix(theta) for theta in thetas] theta_matrices = np.asarray(theta_matrices) # Shape (n_thetas, n_benchmarks) theta_gradient_matrices = np.asarray(theta_gradient_matrices) # Shape (n_thetas, n_gradients, n_benchmarks) # Calculate theta gradient if gradients in ["all", "theta"]: nom_gradients = mdot(theta_gradient_matrices, benchmark_weights) # (n_thetas, n_phys_gradients, n_batch) nuisance_factors = self._calculate_nuisance_factors(nus, benchmark_weights) try: dweight_dtheta = nuisance_factors[:, np.newaxis, :] * nom_gradients except TypeError: dweight_dtheta = nom_gradients else: dweight_dtheta = None # Calculate nu gradient if gradients in ["all", "nu"]: weights_nom = mdot(theta_matrices, benchmark_weights) # Shape (n_thetas, n_batch) nuisance_factor_gradients = np.asarray( [self.nuisance_morpher.calculate_nuisance_factor_gradients(nu, benchmark_weights) for nu in nus] ) # Shape (n_thetas, n_nuisance_gradients, n_batch) dweight_dnu = nuisance_factor_gradients * weights_nom[:, np.newaxis, :] else: dweight_dnu = None if gradients == "theta": return dweight_dtheta elif gradients == "nu": return dweight_dnu return np.concatenate((dweight_dtheta, dweight_dnu), 1)
def weighted_events( self, theta=None, nu=None, start_event=None, end_event=None, derivative=False, generated_close_to=None, n_draws=None, ): """ Returns all events together with the benchmark weights (if theta is None) or weights for a given theta. Parameters ---------- theta : None or ndarray or str, optional If None, the function returns all benchmark weights. If str, the function returns the weights for a given benchmark name. If ndarray, it uses morphing to calculate the weights for this value of theta. Default value: None. nu : None or ndarray, optional If None, the nuisance parameters are set to their nominal values. Otherwise, and if theta is an ndarray, sets the values of the nuisance parameters. start_event : int Index (in the MadMiner file) of the first event to consider. end_event : int Index (in the MadMiner file) of the last unweighted event to consider. derivative : bool, optional If True and if theta is not None, the derivative of the weights with respect to theta are returned. Default value: False. generated_close_to : None or int, optional Only returns benchmarks generated from this benchmark (and background events). Default value: None. n_draws : None or int, optional If not None, returns only this number of events, drawn randomly. Returns ------- x : ndarray Observables with shape `(n_unweighted_samples, n_observables)`. weights : ndarray If theta is None and derivative is False, benchmark weights with shape `(n_unweighted_samples, n_benchmarks)` in pb. If theta is not None and derivative is True, the gradient of the weight for the given parameter with respect to theta with shape `(n_unweighted_samples, n_gradients)` in pb. Otherwise, weights for the given parameter theta with shape `(n_unweighted_samples,)` in pb. """ x, weights_benchmarks = next( self.event_loader( start=start_event, end=end_event, batch_size=None, generated_close_to=generated_close_to, ) ) # Pick events randomly n_events = len(x) if n_draws is not None and n_draws < n_events: idx = np.random.choice(n_events, n_draws, replace=False) x = x[idx] weights_benchmarks = weights_benchmarks[idx] elif n_draws is not None: logger.warning(f"Requested {n_draws} events, but only {n_events} available") # Process and return appropriate weights if theta is None: return x, weights_benchmarks elif isinstance(theta, str): i_benchmark = list(self.benchmarks.keys()).index(theta) return x, weights_benchmarks[:, i_benchmark] elif derivative: dtheta_matrix = self._get_dtheta_benchmark_matrix(theta) gradients_theta = mdot(dtheta_matrix, weights_benchmarks) # (n_gradients, n_samples) gradients_theta = gradients_theta.T return x, gradients_theta else: # TODO: nuisance params if nu is not None: raise NotImplementedError() theta_matrix = self._get_theta_benchmark_matrix(theta) weights_theta = mdot(theta_matrix, weights_benchmarks) return x, weights_theta
def xsec_gradients( self, thetas, nus=None, partition="all", test_split=0.2, validation_split=0.2, gradients="all", batch_size=100000, generated_close_to=None, ): """ Returns the gradient of total cross sections with respect to parameters. Parameters ---------- thetas : list of (ndarray or str), optional If None, the function returns all benchmark cross sections. Otherwise, it returns the cross sections for a series of parameter points that are either given by their benchmark name (as a str), their benchmark index (as an int), or their parameter value (as an ndarray, using morphing). Default value: None. nus : None or list of (None or ndarray), optional If None, the nuisance parameters are set to their nominal values (0), i.e. no systematics are taken into account. Otherwise, the list has to have the same number of elements as thetas, and each entry can specify nuisance parameters at nominal value (None) or a value of the nuisance parameters (ndarray). test_split : float, optional Fraction of events reserved for testing. Default value: 0.2. partition : {"train", "test", "validation", "all"}, optional Which events to use. Default: "all". gradients : {"all", "theta", "nu"}, optional Which gradients to calculate. Default value: "all". batch_size : int, optional Size of the batches of events that are loaded into memory at the same time. Default value: 100000. generated_close_to : None or ndarray, optional If not None, only events originally generated from the closest benchmark to this parameter point will be used. Default value : None. Returns ------- xsecs_gradients : ndarray Calculated cross section gradients in pb with shape (n_gradients,). """ logger.debug("Calculating cross section gradients for thetas = %s and nus = %s", thetas, nus) # Inputs include_nuisance_benchmarks = nus is not None or gradients in ["all", "nu"] if nus is None: nus = [None for _ in thetas] assert len(nus) == len(thetas), "Numbers of thetas and nus don't match!" if gradients not in ["all", "theta", "nu"]: raise RuntimeError("Gradients has to be 'all', 'theta', or 'nu', but got {}".format(gradients)) # Which events to use if partition == "all": start_event, end_event = None, None correction_factor = 1.0 elif partition in ["train", "validation", "test"]: start_event, end_event, correction_factor = self._train_validation_test_split( partition, test_split, validation_split ) else: raise ValueError("Events has to be either 'all', 'train', or 'test', but got {}!".format(partition)) # Theta matrices (translation of benchmarks to theta, at nominal nuisance params) theta_matrices = np.asarray( [self._get_theta_benchmark_matrix(theta) for theta in thetas] ) # shape (n_thetas, n_benchmarks) theta_gradient_matrices = np.asarray( [self._get_dtheta_benchmark_matrix(theta) for theta in thetas] ) # shape (n_thetas, n_gradients, n_benchmarks) # Loop over events xsec_gradients = 0.0 for i_batch, (_, benchmark_weights) in enumerate( self.event_loader( start=start_event, end=end_event, include_nuisance_parameters=include_nuisance_benchmarks, batch_size=batch_size, generated_close_to=generated_close_to, ) ): n_batch, _ = benchmark_weights.shape logger.debug("Batch %s with %s events", i_batch + 1, n_batch) if gradients in ["all", "theta"]: nom_gradients = mdot( theta_gradient_matrices, benchmark_weights ) # Shape (n_thetas, n_phys_gradients, n_batch) nuisance_factors = self._calculate_nuisance_factors(nus, benchmark_weights) # Shape (n_thetas, n_batch) try: dweight_dtheta = nuisance_factors[:, np.newaxis, :] * nom_gradients except TypeError: dweight_dtheta = nom_gradients if gradients in ["all", "nu"]: weights_nom = mdot(theta_matrices, benchmark_weights) # Shape (n_thetas, n_batch) nuisance_factor_gradients = np.asarray( [self.nuisance_morpher.calculate_nuisance_factor_gradients(nu, benchmark_weights) for nu in nus] ) # Shape (n_thetas, n_nuisance_gradients, n_batch) dweight_dnu = nuisance_factor_gradients * weights_nom[:, np.newaxis, :] if gradients == "all": dweight_dall = np.concatenate((dweight_dtheta, dweight_dnu), 1) elif gradients == "theta": dweight_dall = dweight_dtheta elif gradients == "nu": dweight_dall = dweight_dnu xsec_gradients += np.sum(dweight_dall, axis=2) # Correct for not using all events xsec_gradients *= correction_factor return xsec_gradients
def plot_distributions( filename, observables=None, parameter_points=None, uncertainties="nuisance", nuisance_parameters=None, draw_nuisance_toys=None, normalize=False, log=False, observable_labels=None, n_bins=50, line_labels=None, colors=None, linestyles=None, linewidths=1.5, toy_linewidths=0.5, alpha=0.15, toy_alpha=0.75, n_events=None, n_toys=100, n_cols=3, quantiles_for_range=(0.025, 0.975), sample_only_from_closest_benchmark=True, ): """ Plots one-dimensional histograms of observables in a MadMiner file for a given set of benchmarks. Parameters ---------- filename : str Filename of a MadMiner HDF5 file. observables : list of str or None, optional Which observables to plot, given by a list of their names. If None, all observables in the file are plotted. Default value: None. parameter_points : list of (str or ndarray) or None, optional Which parameter points to use for histogramming the data. Given by a list, each element can either be the name of a benchmark in the MadMiner file, or an ndarray specifying any parameter point in a morphing setup. If None, all physics (non-nuisance) benchmarks defined in the MadMiner file are plotted. Default value: None. uncertainties : {"nuisance", "none"}, optional Defines how uncertainty bands are drawn. With "nuisance", the variation in cross section from all nuisance parameters is added in quadrature. With "none", no error bands are drawn. nuisance_parameters : None or list of int, optional If uncertainties is "nuisance", this can restrict which nuisance parameters are used to draw the uncertainty bands. Each entry of this list is the index of one nuisance parameter (same order as in the MadMiner file). draw_nuisance_toys : None or int, optional If not None and uncertainties is "nuisance", sets the number of nuisance toy distributions that are drawn (in addition to the error bands). normalize : bool, optional Whether the distribution is normalized to the total cross section. Default value: False. log : bool, optional Whether to draw the y axes on a logarithmic scale. Defaul value: False. observable_labels : None or list of (str or None), optional x-axis labels naming the observables. If None, the observable names from the MadMiner file are used. Default value: None. n_bins : int, optional Number of histogram bins. Default value: 50. line_labels : None or list of (str or None), optional Labels for the different parameter points. If None and if parameter_points is None, the benchmark names from the MadMiner file are used. Default value: None. colors : None or str or list of str, optional Matplotlib line (and error band) colors for the distributions. If None, uses default colors. Default value: None. linestyles : None or str or list of str, optional Matplotlib line styles for the distributions. If None, uses default linestyles. Default value: None. linewidths : float or list of float, optional Line widths for the contours. Default value: 1.5. toy_linewidths : float or list of float or None, optional Line widths for the toy replicas, if uncertainties is "nuisance" and draw_nuisance_toys is not None. If None, linewidths is used. Default value: 1. alpha : float, optional alpha value for the uncertainty bands. Default value: 0.25. toy_alpha : float, optional alpha value for the toy replicas, if uncertainties is "nuisance" and draw_nuisance_toys is not None. Default value: 0.75. n_events : None or int, optional If not None, sets the number of events from the MadMiner file that will be analyzed and plotted. Default value: None. n_toys : int, optional Number of toy nuisance parameter vectors used to estimate the systematic uncertainties. Default value: 100. n_cols : int, optional Number of columns of subfigures in the plot. Default value: 3. quantiles_for_range : tuple of two float, optional Tuple `(min_quantile, max_quantile)` that defines how the observable range is determined for each panel. Default: (0.025, 0.075). Returns ------- figure : Figure Plot as Matplotlib Figure instance. """ # Load data sa = SampleAugmenter(filename, include_nuisance_parameters=True) if uncertainties == "nuisance": nuisance_morpher = NuisanceMorpher( sa.nuisance_parameters, list(sa.benchmarks.keys()), reference_benchmark=sa.reference_benchmark ) # Default settings if parameter_points is None: parameter_points = [] for key, is_nuisance in zip(sa.benchmarks, sa.benchmark_is_nuisance): if not is_nuisance: parameter_points.append(key) if line_labels is None: line_labels = parameter_points n_parameter_points = len(parameter_points) if colors is None: colors = ["C" + str(i) for i in range(10)] * (n_parameter_points // 10 + 1) elif not isinstance(colors, list): colors = [colors for _ in range(n_parameter_points)] if linestyles is None: linestyles = ["solid", "dashed", "dotted", "dashdot"] * (n_parameter_points // 4 + 1) elif not isinstance(linestyles, list): linestyles = [linestyles for _ in range(n_parameter_points)] if not isinstance(linewidths, list): linewidths = [linewidths for _ in range(n_parameter_points)] if toy_linewidths is None: toy_linewidths = linewidths if not isinstance(toy_linewidths, list): toy_linewidths = [toy_linewidths for _ in range(n_parameter_points)] # Observables observable_indices = [] if observables is None: observable_indices = list(range(len(sa.observables))) else: all_observables = list(sa.observables.keys()) for obs in observables: try: observable_indices.append(all_observables.index(str(obs))) except ValueError: logging.warning("Ignoring unknown observable %s", obs) logger.debug("Observable indices: %s", observable_indices) n_observables = len(observable_indices) if observable_labels is None: all_observables = list(sa.observables.keys()) observable_labels = [all_observables[obs] for obs in observable_indices] # Parse thetas theta_values = [sa._get_theta_value(theta) for theta in parameter_points] theta_matrices = [sa._get_theta_benchmark_matrix(theta) for theta in parameter_points] logger.debug("Calculated %s theta matrices", len(theta_matrices)) # Get event data (observations and weights) all_x, all_weights_benchmarks = sa.weighted_events(generated_close_to=None) logger.debug("Loaded raw data with shapes %s, %s", all_x.shape, all_weights_benchmarks.shape) indiv_x, indiv_weights_benchmarks = [], [] if sample_only_from_closest_benchmark: for theta in theta_values: this_x, this_weights = sa.weighted_events(generated_close_to=theta) indiv_x.append(this_x) indiv_weights_benchmarks.append(this_weights) # Remove negative weights sane_event_filter = np.all(all_weights_benchmarks >= 0.0, axis=1) n_events_before = all_weights_benchmarks.shape[0] all_x = all_x[sane_event_filter] all_weights_benchmarks = all_weights_benchmarks[sane_event_filter] n_events_removed = n_events_before - all_weights_benchmarks.shape[0] if int(np.sum(sane_event_filter, dtype=np.int)) < len(sane_event_filter): logger.warning("Removed %s / %s events with negative weights", n_events_removed, n_events_before) for i, (x, weights) in enumerate(zip(indiv_x, indiv_weights_benchmarks)): sane_event_filter = np.all(weights >= 0.0, axis=1) indiv_x[i] = x[sane_event_filter] indiv_weights_benchmarks[i] = weights[sane_event_filter] # Shuffle events all_x, all_weights_benchmarks = shuffle(all_x, all_weights_benchmarks) for i, (x, weights) in enumerate(zip(indiv_x, indiv_weights_benchmarks)): indiv_x[i], indiv_weights_benchmarks[i] = shuffle(x, weights) # Only analyze n_events if n_events is not None and n_events < all_x.shape[0]: logger.debug("Only analyzing first %s / %s events", n_events, all_x.shape[0]) all_x = all_x[:n_events] all_weights_benchmarks = all_weights_benchmarks[:n_events] for i, (x, weights) in enumerate(zip(indiv_x, indiv_weights_benchmarks)): indiv_x[i] = x[:n_events] indiv_weights_benchmarks[i] = weights[:n_events] if uncertainties != "nuisance": n_toys = 0 n_nuisance_toys_drawn = 0 if draw_nuisance_toys is not None: n_nuisance_toys_drawn = draw_nuisance_toys # Nuisance parameters nuisance_toy_factors = [] if uncertainties == "nuisance": n_nuisance_params = sa.n_nuisance_parameters if not n_nuisance_params > 0: raise RuntimeError("Cannot draw systematic uncertainties -- no nuisance parameters found!") logger.debug("Drawing nuisance toys") nuisance_toys = np.random.normal(loc=0.0, scale=1.0, size=n_nuisance_params * n_toys) nuisance_toys = nuisance_toys.reshape(n_toys, n_nuisance_params) # Restrict nuisance parameters if nuisance_parameters is not None: for i in range(n_nuisance_params): if i not in nuisance_parameters: nuisance_toys[:, i] = 0.0 logger.debug("Drew %s toy values for nuisance parameters", n_toys * n_nuisance_params) nuisance_toy_factors = np.array( [ nuisance_morpher.calculate_nuisance_factors(nuisance_toy, all_weights_benchmarks) for nuisance_toy in nuisance_toys ] ) # Shape (n_toys, n_events) nuisance_toy_factors = sanitize_array(nuisance_toy_factors, min_value=1.0e-2, max_value=100.0) # Shape (n_toys, n_events) # Preparing plot n_rows = (n_observables + n_cols - 1) // n_cols n_events_for_range = 10000 if n_events is None else min(10000, n_events) fig = plt.figure(figsize=(4.0 * n_cols, 4.0 * n_rows)) for i_panel, (i_obs, xlabel) in enumerate(zip(observable_indices, observable_labels)): logger.debug("Plotting panel %s: observable %s, label %s", i_panel, i_obs, xlabel) # Figure out x range xmins, xmaxs = [], [] for theta_matrix in theta_matrices: x_small = all_x[:n_events_for_range] weights_small = mdot(theta_matrix, all_weights_benchmarks[:n_events_for_range]) xmin = weighted_quantile(x_small[:, i_obs], quantiles_for_range[0], weights_small) xmax = weighted_quantile(x_small[:, i_obs], quantiles_for_range[1], weights_small) xwidth = xmax - xmin xmin -= xwidth * 0.1 xmax += xwidth * 0.1 xmin = max(xmin, np.min(all_x[:, i_obs])) xmax = min(xmax, np.max(all_x[:, i_obs])) xmins.append(xmin) xmaxs.append(xmax) xmin = min(xmins) xmax = max(xmaxs) x_range = (xmin, xmax) logger.debug("Ranges for observable %s: min = %s, max = %s", xlabel, xmins, xmaxs) # Subfigure ax = plt.subplot(n_rows, n_cols, i_panel + 1) # Calculate histograms bin_edges = None histos = [] histos_up = [] histos_down = [] histos_toys = [] for i_theta, theta_matrix in enumerate(theta_matrices): theta_weights = mdot(theta_matrix, all_weights_benchmarks) # Shape (n_events,) if sample_only_from_closest_benchmark: indiv_theta_weights = mdot(theta_matrix, indiv_weights_benchmarks[i_theta]) # Shape (n_events,) histo, bin_edges = np.histogram( indiv_x[i_theta][:, i_obs], bins=n_bins, range=x_range, weights=indiv_theta_weights, density=normalize, ) else: histo, bin_edges = np.histogram( all_x[:, i_obs], bins=n_bins, range=x_range, weights=theta_weights, density=normalize ) histos.append(histo) if uncertainties == "nuisance": histos_toys_this_theta = [] for i_toy, nuisance_toy_factors_this_toy in enumerate(nuisance_toy_factors): toy_histo, _ = np.histogram( all_x[:, i_obs], bins=n_bins, range=x_range, weights=theta_weights * nuisance_toy_factors_this_toy, density=normalize, ) histos_toys_this_theta.append(toy_histo) histos_up.append(np.percentile(histos_toys_this_theta, 84.0, axis=0)) histos_down.append(np.percentile(histos_toys_this_theta, 16.0, axis=0)) histos_toys.append(histos_toys_this_theta[:n_nuisance_toys_drawn]) # Draw error bands if uncertainties == "nuisance": for histo_up, histo_down, lw, color, label, ls in zip( histos_up, histos_down, linewidths, colors, line_labels, linestyles ): bin_edges_ = np.repeat(bin_edges, 2)[1:-1] histo_down_ = np.repeat(histo_down, 2) histo_up_ = np.repeat(histo_up, 2) plt.fill_between(bin_edges_, histo_down_, histo_up_, facecolor=color, edgecolor="none", alpha=alpha) # Draw some toys for histo_toys, lw, color, ls in zip(histos_toys, toy_linewidths, colors, linestyles): for k in range(n_nuisance_toys_drawn): bin_edges_ = np.repeat(bin_edges, 2)[1:-1] histo_ = np.repeat(histo_toys[k], 2) plt.plot(bin_edges_, histo_, color=color, alpha=toy_alpha, lw=lw, ls=ls) # Draw central lines for histo, lw, color, label, ls in zip(histos, linewidths, colors, line_labels, linestyles): bin_edges_ = np.repeat(bin_edges, 2)[1:-1] histo_ = np.repeat(histo, 2) plt.plot(bin_edges_, histo_, color=color, lw=lw, ls=ls, label=label, alpha=1.0) plt.legend() plt.xlabel(xlabel) if normalize: plt.ylabel("Normalized distribution") else: plt.ylabel(r"$\frac{d\sigma}{dx}$ [pb / bin]") plt.xlim(x_range[0], x_range[1]) if log: ax.set_yscale("log", nonposy="clip") else: plt.ylim(0.0, None) plt.tight_layout() return fig
def plot_uncertainty( filename, theta, observable, obs_label, obs_range, n_bins=50, nuisance_parameters=None, n_events=None, n_toys=100, linecolor="black", bandcolor1="#CC002E", bandcolor2="orange", ratio_range=(0.8, 1.2), ): """ Plots absolute and relative uncertainty bands in a histogram of one observable in a MadMiner file. Parameters ---------- filename : str Filename of a MadMiner HDF5 file. theta : ndarray, optional Which parameter points to use for histogramming the data. observable : str Which observable to plot, given by its name in the MadMiner file. obs_label : str x-axis label naming the observable. obs_range : tuple of two float Range to be plotted for the observable. n_bins : int Number of bins. Default value: 50. nuisance_parameters : None or list of int, optional This can restrict which nuisance parameters are used to draw the uncertainty bands. Each entry of this list is the index of one nuisance parameter (same order as in the MadMiner file). n_events : None or int, optional If not None, sets the number of events from the MadMiner file that will be analyzed and plotted. Default value: None. n_toys : int, optional Number of toy nuisance parameter vectors used to estimate the systematic uncertainties. Default value: 100. linecolor : str, optional Line color for central prediction. Default value: "black". bandcolor1 : str, optional Error band color for 1 sigma uncertainty. Default value: "#CC002E". bandcolor2 : str, optional Error band color for 2 sigma uncertainty. Default value: "orange". ratio_range : tuple of two floar y-axis range for the plots of the ratio to the central prediction. Default value: (0.8, 1.2). Returns ------- figure : Figure Plot as Matplotlib Figure instance. """ # Load data sa = SampleAugmenter(filename, include_nuisance_parameters=True) nuisance_morpher = NuisanceMorpher( sa.nuisance_parameters, list(sa.benchmarks.keys()), reference_benchmark=sa.reference_benchmark ) # Observable index obs_idx = list(sa.observables.keys()).index(observable) # Get event data (observations and weights) x, weights_benchmarks = sa.weighted_events() x = x[:, obs_idx] # Theta matrix theta_matrix = sa._get_theta_benchmark_matrix(theta) weights = mdot(theta_matrix, weights_benchmarks) # Remove negative weights x = x[weights >= 0.0] weights_benchmarks = weights_benchmarks[weights >= 0.0] weights = weights[weights >= 0.0] # Shuffle events x, weights, weights_benchmarks = shuffle(x, weights, weights_benchmarks) # Only analyze n_events if n_events is not None and n_events < x.shape[0]: x = x[:n_events] weights_benchmarks = weights_benchmarks[:n_events] weights = weights[:n_events] # Nuisance parameters n_nuisance_params = sa.n_nuisance_parameters nuisance_toys = np.random.normal(loc=0.0, scale=1.0, size=n_nuisance_params * n_toys) nuisance_toys = nuisance_toys.reshape(n_toys, n_nuisance_params) # Restrict nuisance parameters if nuisance_parameters is not None: for i in range(n_nuisance_params): if i not in nuisance_parameters: nuisance_toys[:, i] = 0.0 nuisance_toy_factors = np.array( [ nuisance_morpher.calculate_nuisance_factors(nuisance_toy, weights_benchmarks) for nuisance_toy in nuisance_toys ] ) # Shape (n_toys, n_events) nuisance_toy_factors = sanitize_array(nuisance_toy_factors, min_value=1.0e-2, max_value=100.0) # Shape (n_toys, n_events) # Calculate histogram for central prediction, not normalized histo, bin_edges = np.histogram(x, bins=n_bins, range=obs_range, weights=weights, density=False) # Calculate toy histograms, not normalized histos_toys_this_theta = [] for i_toy, nuisance_toy_factors_this_toy in enumerate(nuisance_toy_factors): toy_histo, _ = np.histogram( x, bins=n_bins, range=obs_range, weights=weights * nuisance_toy_factors_this_toy, density=False ) histos_toys_this_theta.append(toy_histo) histo_plus2sigma = np.percentile(histos_toys_this_theta, 97.5, axis=0) histo_plus1sigma = np.percentile(histos_toys_this_theta, 84.0, axis=0) histo_minus1sigma = np.percentile(histos_toys_this_theta, 16.0, axis=0) histo_minus2sigma = np.percentile(histos_toys_this_theta, 2.5, axis=0) # Calculate histogram for central prediction, normalized histo_norm, bin_edges_norm = np.histogram(x, bins=n_bins, range=obs_range, weights=weights, density=True) # Calculate toy histograms, normalized histos_toys_this_theta = [] for i_toy, nuisance_toy_factors_this_toy in enumerate(nuisance_toy_factors): toy_histo, _ = np.histogram( x, bins=n_bins, range=obs_range, weights=weights * nuisance_toy_factors_this_toy, density=True ) histos_toys_this_theta.append(toy_histo) histo_plus2sigma_norm = np.percentile(histos_toys_this_theta, 97.5, axis=0) histo_plus1sigma_norm = np.percentile(histos_toys_this_theta, 84.0, axis=0) histo_minus1sigma_norm = np.percentile(histos_toys_this_theta, 16.0, axis=0) histo_minus2sigma_norm = np.percentile(histos_toys_this_theta, 2.5, axis=0) # Prepare plotting def plot_mc(edges, histo_central, histo_m2, histo_m1, histo_p1, histo_p2, relative=False): bin_edges_ = np.repeat(edges, 2)[1:-1] histo_ = np.repeat(histo_central, 2) histo_m2_ = np.repeat(histo_m2, 2) histo_m1_ = np.repeat(histo_m1, 2) histo_p1_ = np.repeat(histo_p1, 2) histo_p2_ = np.repeat(histo_p2, 2) if relative: histo_m2_ /= histo_ histo_m1_ /= histo_ histo_p1_ /= histo_ histo_p2_ /= histo_ histo_ /= histo_ plt.fill_between(bin_edges_, histo_m2_, histo_p2_, facecolor=bandcolor2, edgecolor="none") plt.fill_between(bin_edges_, histo_m1_, histo_p1_, facecolor=bandcolor1, edgecolor="none") plt.plot(bin_edges_, histo_, color=linecolor, lw=1.5, ls="-") # Make plot fig = plt.figure(figsize=(10, 7)) gs = gridspec.GridSpec(2, 2, height_ratios=[2, 1]) # MC, absolute residuals ax = plt.subplot(gs[2]) plot_mc(bin_edges, histo, histo_minus2sigma, histo_minus1sigma, histo_plus1sigma, histo_plus2sigma, relative=True) plt.xlabel(obs_label) plt.ylabel(r"Relative to central pred.") plt.xlim(obs_range[0], obs_range[1]) plt.ylim(ratio_range[0], ratio_range[1]) # MC, absolute ax = plt.subplot(gs[0], sharex=ax) plot_mc(bin_edges, histo, histo_minus2sigma, histo_minus1sigma, histo_plus1sigma, histo_plus2sigma) plt.ylabel(r"Differential cross section [pb/bin]") plt.ylim(0.0, None) plt.setp(ax.get_xticklabels(), visible=False) # MC, relative residuals ax = plt.subplot(gs[3]) plot_mc( bin_edges_norm, histo_norm, histo_minus2sigma_norm, histo_minus1sigma_norm, histo_plus1sigma_norm, histo_plus2sigma_norm, relative=True, ) plt.xlabel(r"$p_{T,\gamma}$ [GeV]") plt.ylabel(r"Relative to central pred.") plt.xlim(obs_range[0], obs_range[1]) plt.ylim(ratio_range[0], ratio_range[1]) # MC, relative ax = plt.subplot(gs[1], sharex=ax) plot_mc( bin_edges_norm, histo_norm, histo_minus2sigma_norm, histo_minus1sigma_norm, histo_plus1sigma_norm, histo_plus2sigma_norm, ) plt.ylabel(r"Normalized distribution") plt.ylim(0.0, None) plt.setp(ax.get_xticklabels(), visible=False) # Return plt.tight_layout() return fig
def xsecs(self, thetas=None, nus=None, events="all", test_split=0.2, include_nuisance_benchmarks=True, batch_size=100000): """ Returns the total cross sections for benchmarks or parameter points. Parameters ---------- thetas : None or list of (ndarray or str), optional If None, the function returns all benchmark cross sections. Otherwise, it returns the cross sections for a series of parameter points that are either given by their benchmark name (as a str), their benchmark index (as an int), or their parameter value (as an ndarray, using morphing). Default value: None. nus : None or list of (None or ndarray), optional If None, the nuisance parameters are set to their nominal values (0), i.e. no systematics are taken into account. Otherwise, the list has to have the same number of elements as thetas, and each entry can specify nuisance parameters at nominal value (None) or a value of the nuisance parameters (ndarray). include_nuisance_benchmarks : bool, optional Whether to include nuisance benchmarks if thetas is None. Default value: True. test_split : float, optional Fraction of events reserved for testing. Default value: 0.2. events : {"train", "test", "all"}, optional Which events to use. Default: "all". batch_size : int, optional Size of the batches of events that are loaded into memory at the same time. Default value: 100000. Returns ------- xsecs : ndarray Calculated cross sections in pb. xsec_uncertainties : ndarray Cross-section uncertainties in pb. Basically calculated as sum(weights**2)**0.5. """ logger.debug("Calculating cross sections for thetas = %s and nus = %s", thetas, nus) # Inputs if thetas is not None: include_nuisance_benchmarks = True if thetas is not None: if nus is None: nus = [None for _ in thetas] assert len(nus) == len( thetas), "Numbers of thetas and nus don't match!" # Which events to use if events == "all": start_event, end_event = None, None correction_factor = 1.0 elif events == "train": start_event, end_event, correction_factor = self._train_test_split( True, test_split) elif events == "test": start_event, end_event, correction_factor = self._train_test_split( False, test_split) else: raise ValueError( "Events has to be either 'all', 'train', or 'test', but got {}!" .format(events)) # Theta matrices (translation of benchmarks to theta, at nominal nuisance params) theta_matrices = [ self._get_theta_benchmark_matrix(theta) for theta in thetas ] theta_matrices = np.asarray( theta_matrices) # Shape (n_thetas, n_benchmarks) # Loop over events xsecs = 0.0 xsec_uncertainties = 0.0 for i_batch, (_, benchmark_weights) in enumerate( madminer_event_loader( self.madminer_filename, start=start_event, end=end_event, include_nuisance_parameters=include_nuisance_benchmarks, benchmark_is_nuisance=self.benchmark_is_nuisance, batch_size=batch_size, )): n_batch, _ = benchmark_weights.shape logger.debug("Batch %s with %s events", i_batch + 1, n_batch) # Benchmark xsecs if thetas is None: xsecs += np.sum(benchmark_weights, axis=0) xsec_uncertainties += np.sum(benchmark_weights * benchmark_weights, axis=0) # xsecs at given parame ters(theta, nu) else: # Weights at nominal nuisance params (nu=0) weights_nom = mdot( theta_matrices, benchmark_weights) # Shape (n_thetas, n_batch) weights_sq_nom = mdot(theta_matrices, benchmark_weights * benchmark_weights) # same logger.debug("Nominal weights: %s", weights_nom) # Effect of nuisance parameters nuisance_factors = self._calculate_nuisance_factors( nus, benchmark_weights) weights = nuisance_factors * weights_nom weights_sq = nuisance_factors * weights_sq_nom logger.debug("Nuisance factors: %s", nuisance_factors) # Sum up xsecs += np.sum(weights, axis=1) xsec_uncertainties += np.sum(weights_sq, axis=1) xsec_uncertainties = xsec_uncertainties**0.5 # Correct for not using all events xsecs *= correction_factor xsec_uncertainties *= correction_factor logger.debug("xsecs and uncertainties [pb]:") for this_xsec, this_uncertainty in zip(xsecs, xsec_uncertainties): logger.debug(" (%4f +/- %4f) pb (%4f %%)", this_xsec, this_uncertainty, 100 * this_uncertainty / this_xsec) return xsecs, xsec_uncertainties
def weighted_events(self, theta=None, nu=None, start_event=None, end_event=None, derivative=False): """ Returns all events together with the benchmark weights (if theta is None) or weights for a given theta. Parameters ---------- theta : None or ndarray or str, optional If None, the function returns all benchmark weights. If str, the function returns the weights for a given benchmark name. If ndarray, it uses morphing to calculate the weights for this value of theta. Default value: None. nu : None or ndarray, optional If None, the nuisance parameters are set to their nominal values. Otherwise, and if theta is an ndarray, sets the values of the nuisance parameters. start_event : int Index (in the MadMiner file) of the first event to consider. end_event : int Index (in the MadMiner file) of the last unweighted event to consider. derivative : bool, optional If True and if theta is not None, the derivative of the weights with respect to theta are returned. Default value: False. Returns ------- x : ndarray Observables with shape `(n_unweighted_samples, n_observables)`. weights : ndarray If theta is None and derivative is False, benchmark weights with shape `(n_unweighted_samples, n_benchmarks)` in pb. If theta is not None and derivative is True, the gradient of the weight for the given parameter with respect to theta with shape `(n_unweighted_samples, n_gradients)` in pb. Otherwise, weights for the given parameter theta with shape `(n_unweighted_samples,)` in pb. """ x, weights_benchmarks = next( self.event_loader(batch_size=None, start=start_event, end=end_event)) if theta is None: return x, weights_benchmarks elif isinstance(theta, six.string_types): i_benchmark = list(self.benchmarks.keys()).index(theta) return x, weights_benchmarks[:, i_benchmark] elif derivative: dtheta_matrix = self._get_dtheta_benchmark_matrix(theta) gradients_theta = mdot( dtheta_matrix, weights_benchmarks) # (n_gradients, n_samples) gradients_theta = gradients_theta.T return x, gradients_theta else: # TODO: nuisance params if nu is not None: raise NotImplementedError theta_matrix = self._get_theta_benchmark_matrix(theta) weights_theta = mdot(theta_matrix, weights_benchmarks) return x, weights_theta