def plot_flatness_by_particle(labels, predictions_dict, spectator, spectator_name, predictions_dict_comparison=None, names_algorithms=['MVA', 'Baseline'], weights=None, bins_number=30, ignored_sideband=0.1, thresholds=None, cuts_values=False, ncol=1): plt.figure(figsize=(22, 20)) for n, (name, label) in enumerate(names_labels_correspondence.items()): plt.subplot(3, 2, n + 1) mask =labels == label legends = [] for preds, name_algo in zip([predictions_dict, predictions_dict_comparison], names_algorithms): if preds is None: continue probs = preds[label][mask] if cuts_values: thresholds_values = cut_values else: thresholds_values = [weighted_quantile(probs, quantiles=1 - eff / 100., sample_weight=None if weights is None else weights[mask]) for eff in thresholds] eff = get_efficiencies(probs, spectator[mask], sample_weight=None if weights is None else weights[mask], bins_number=bins_number, errors=True, ignored_sideband=ignored_sideband, thresholds=thresholds_values) for thr in thresholds_values: eff[thr] = (eff[thr][0], 100*numpy.array(eff[thr][1]), 100*numpy.array(eff[thr][2]), eff[thr][3]) plot_fig = ErrorPlot(eff) plot_fig.xlabel = '{} {}'.format(name, spectator_name) plot_fig.ylabel = 'Efficiency' plot_fig.title = name plot_fig.ylim = (0, 100) plot_fig.plot(fontsize=22) plt.xticks(fontsize=12), plt.yticks(fontsize=12) legends.append(['{} Eff {}%'.format(thr, name_algo) for thr in thresholds]) plt.legend(numpy.concatenate(legends), loc='best', fontsize=12, framealpha=0.5, ncol=ncol)
def plot_flatness_particle(labels, predictions_dict, spectator, spectator_name, particle_name, weights=None, bins_number=30, ignored_sideband=0.1, thresholds=None, cuts_values=False): plt.figure(figsize=(18, 22)) for n, (name, label) in enumerate(names_labels_correspondence.items()): plt.subplot(3, 2, n + 1) mask = labels == names_labels_correspondence[particle_name] probs = predictions_dict[label][mask] mask_signal = labels == label probs_signal = predictions_dict[label][mask_signal] if cuts_values: thresholds_values = cut_values else: thresholds_values = [weighted_quantile(probs_signal, quantiles=1 - eff / 100., sample_weight=None if weights is None else weights[mask_signal]) for eff in thresholds] eff = get_efficiencies(probs, spectator[mask], sample_weight=None if weights is None else weights[mask], bins_number=bins_number, errors=True, ignored_sideband=ignored_sideband, thresholds=thresholds_values) for thr in thresholds_values: eff[thr] = (eff[thr][0], 100*numpy.array(eff[thr][1]), 100*numpy.array(eff[thr][2]), eff[thr][3]) plot_fig = ErrorPlot(eff) plot_fig.xlabel = '{} {}'.format(particle_name, spectator_name) plot_fig.ylabel = 'Efficiency' plot_fig.title = 'MVA {}'.format(name) plot_fig.ylim = (0, 100) plot_fig.plot(fontsize=22) plt.xticks(fontsize=12), plt.yticks(fontsize=12) if not cuts_values: plt.legend(['Signal Eff {}%'.format(thr) for thr in thresholds], loc='best', fontsize=18, framealpha=0.5)
def test_calc(): prediction = numpy.random.random(10000) iron = utils.Flattener(prediction) assert numpy.allclose(numpy.histogram(iron(prediction), normed=True, bins=30)[0], numpy.ones(30), rtol=1e-02) x, y, yerr, xerr = utils.calc_hist_with_errors(iron(prediction), bins=30, x_range=(0, 1)) assert numpy.allclose(y, numpy.ones(len(y)), rtol=1e-02) width = 1. / 60 means = numpy.linspace(width, 1 - width, 30) assert numpy.allclose(x, means) assert numpy.allclose(xerr, numpy.zeros(len(xerr)) + width) assert numpy.allclose(yerr, numpy.zeros(len(yerr)) + yerr[0], rtol=1e-2) random_labels = numpy.random.choice(2, size=10000) (tpr, tnr), _, _ = utils.calc_ROC(prediction, random_labels) # checking for random classifier assert numpy.max(abs(1 - tpr - tnr)) < 0.05 # checking efficiencies for random mass, random prediction mass = numpy.random.random(10000) result = utils.get_efficiencies(prediction, mass) for threshold, (xval, yval) in result.items(): assert ((yval + threshold - 1)**2).mean() < 0.1
def plot_flatness_by_particle(labels, predictions_dict, spectator, spectator_name, predictions_dict_comparison=None, names_algorithms=['MVA', 'Baseline'], weights=None, bins_number=30, ignored_sideband=0.1, thresholds=None, cuts_values=False, ncol=1): plt.figure(figsize=(22, 20)) for n, (name, label) in enumerate(names_labels_correspondence.items()): plt.subplot(3, 2, n + 1) mask = labels == label legends = [] for preds, name_algo in zip( [predictions_dict, predictions_dict_comparison], names_algorithms): if preds is None: continue probs = preds[label][mask] if cuts_values: thresholds_values = cut_values else: thresholds_values = [ weighted_quantile(probs, quantiles=1 - eff / 100., sample_weight=None if weights is None else weights[mask]) for eff in thresholds ] eff = get_efficiencies( probs, spectator[mask], sample_weight=None if weights is None else weights[mask], bins_number=bins_number, errors=True, ignored_sideband=ignored_sideband, thresholds=thresholds_values) for thr in thresholds_values: eff[thr] = (eff[thr][0], 100 * numpy.array(eff[thr][1]), 100 * numpy.array(eff[thr][2]), eff[thr][3]) plot_fig = ErrorPlot(eff) plot_fig.xlabel = '{} {}'.format(name, spectator_name) plot_fig.ylabel = 'Efficiency' plot_fig.title = name plot_fig.ylim = (0, 100) plot_fig.plot(fontsize=22) plt.xticks(fontsize=12), plt.yticks(fontsize=12) legends.append( ['{} Eff {}%'.format(thr, name_algo) for thr in thresholds]) plt.legend(numpy.concatenate(legends), loc='best', fontsize=12, framealpha=0.5, ncol=ncol)
def plot_flatness_particle(labels, predictions_dict, spectator, spectator_name, particle_name, weights=None, bins_number=30, ignored_sideband=0.1, thresholds=None, cuts_values=False): plt.figure(figsize=(18, 22)) for n, (name, label) in enumerate(names_labels_correspondence.items()): plt.subplot(3, 2, n + 1) mask = labels == names_labels_correspondence[particle_name] probs = predictions_dict[label][mask] mask_signal = labels == label probs_signal = predictions_dict[label][mask_signal] if cuts_values: thresholds_values = cut_values else: thresholds_values = [ weighted_quantile(probs_signal, quantiles=1 - eff / 100., sample_weight=None if weights is None else weights[mask_signal]) for eff in thresholds ] eff = get_efficiencies( probs, spectator[mask], sample_weight=None if weights is None else weights[mask], bins_number=bins_number, errors=True, ignored_sideband=ignored_sideband, thresholds=thresholds_values) for thr in thresholds_values: eff[thr] = (eff[thr][0], 100 * numpy.array(eff[thr][1]), 100 * numpy.array(eff[thr][2]), eff[thr][3]) plot_fig = ErrorPlot(eff) plot_fig.xlabel = '{} {}'.format(particle_name, spectator_name) plot_fig.ylabel = 'Efficiency' plot_fig.title = 'MVA {}'.format(name) plot_fig.ylim = (0, 100) plot_fig.plot(fontsize=22) plt.xticks(fontsize=12), plt.yticks(fontsize=12) if not cuts_values: plt.legend(['Signal Eff {}%'.format(thr) for thr in thresholds], loc='best', fontsize=18, framealpha=0.5)
def test_calc(): prediction = numpy.random.random(10000) iron = utils.Flattener(prediction) assert numpy.allclose(numpy.histogram(iron(prediction), normed=True, bins=30)[0], numpy.ones(30), rtol=1e-02) x, y, y_err, x_err = utils.calc_hist_with_errors(iron(prediction), bins=30, x_range=(0, 1)) assert numpy.allclose(y, numpy.ones(len(y)), rtol=1e-02) width = 1. / 60 means = numpy.linspace(width, 1 - width, 30) assert numpy.allclose(x, means) assert numpy.allclose(x_err, numpy.zeros(len(x_err)) + width) assert numpy.allclose(y_err, numpy.zeros(len(y_err)) + y_err[0], rtol=1e-2) random_labels = numpy.random.choice(2, size=10000) (tpr, tnr), _, _ = utils.calc_ROC(prediction, random_labels) # checking for random classifier assert numpy.max(abs(1 - tpr - tnr)) < 0.05 # checking efficiencies for random mass, random prediction mass = numpy.random.random(10000) result = utils.get_efficiencies(prediction, mass) for threshold, (xval, yval) in result.items(): assert ((yval + threshold - 1) ** 2).mean() < 0.1
def plot_flatness_by_particle(labels, predictions, spectator, spectator_name, predictions_comparison=None, names_algorithms=['MVA', 'Baseline'], for_particle=None, weights=None, bins_number=30, ignored_sideband=0.1, thresholds=None, n_col=1): """ Build a flatness-plot, which demonstrates the dependency between efficiency and some observable. :param labels: [n_samples], contains targets :param predictions: [n_samples, n_particle_types] with predictions of an algorithm :param spectator: [n_samples], values of spectator variable :param spectator_name: str, name shown on the plot :param predictions_comparison: [n_samples, n_particle types], optionally for comparison this may be provided :param names_algorithms: names for compared algorithms :param weights: [n_samples], optional :param bins_number: int, :param ignored_sideband: fraction of ignored sidebands :param thresholds: efficiencies, for which flatness is drawn :param n_col: number of columns in legend. """ plt.figure(figsize=(22, 24)) if predictions_comparison is not None: colors = ['blue', 'green'] markers = ['o', 's', 'v', 'o', 's', 'v'] else: colors = [None, None] markers = ['o'] * len(thresholds) for n, (particle_name, label) in enumerate(names_labels_correspondence.items()): plt.subplot(3, 2, n + 1) title = '{} algorithm'.format(particle_name) xlim_all = (1e10, -1e10) ylim_all = (20, -1e8) legends = [] for preds, algo_name, color in zip( [predictions, predictions_comparison], names_algorithms, colors): if preds is None: continue particle_mask = labels == label particle_probs = preds[particle_mask, label] particle_weights = None if weights is None else weights[ particle_mask] thresholds_values = [ weighted_quantile(particle_probs, quantiles=1 - eff / 100., sample_weight=particle_weights) for eff in thresholds ] if for_particle is not None: particle_mask = labels == names_labels_correspondence[ for_particle] particle_probs = preds[particle_mask, label] particle_weights = None if weights is None else weights[ particle_mask] title = '{} algorithm for {}'.format(particle_name, for_particle) eff = get_efficiencies(particle_probs, spectator[particle_mask], sample_weight=particle_weights, bins_number=bins_number, errors=True, ignored_sideband=ignored_sideband, thresholds=thresholds_values) for thr in thresholds_values: eff[thr] = (eff[thr][0], 100 * numpy.array(eff[thr][1]), 100 * numpy.array(eff[thr][2]), eff[thr][3]) xlim, ylim = compute_limits_and_plot_errorbar(eff, markers, color=color) plt.xlabel('{} {}\n\n'.format(particle_name, spectator_name), fontsize=22) plt.ylabel('Efficiency', fontsize=22) plt.title('\n\n'.format(title), fontsize=22) plt.xticks(fontsize=12), plt.yticks(fontsize=12) legends.append( ['{} Eff {}%'.format(algo_name, thr) for thr in thresholds]) plt.grid(True) xlim_all = (min(xlim_all[0], xlim[0]), max(xlim_all[1], xlim[1])) ylim_all = (min(ylim_all[0], ylim[0]), max(ylim_all[1], ylim[1])) plt.legend(numpy.concatenate(legends), loc='best', fontsize=16, framealpha=0.5, ncol=n_col) plt.xlim(xlim_all[0], xlim_all[1]) plt.ylim(ylim_all[0], ylim_all[1])
def flatness_eta_figure(proba, proba_baseline, eta, track_name, particle_name, save_path=None, show=False): """ Plot signal efficiency vs pseudo rapidity figure. Parameters ---------- proba : array_like Predicted probabilities with array shape = [n_samples]. probas_baseline : array_like Baseline predicted probabilities with array shape = [n_samples]. eta : array_like Pseudo rapidity values with array shape = [n_samples]. track_name : string The track name. particle_name : string The particle name. save_path : string Path to a directory where the figure will saved. If None the figure will not be saved. show : boolean If true the figure will be displayed. """ thresholds = numpy.percentile(proba, 100 - numpy.array([20, 50, 80])) thresholds_baseline = numpy.percentile(proba_baseline, 100 - numpy.array([20, 50, 80])) eff = get_efficiencies(proba, eta, bins_number=30, errors=True, ignored_sideband=0.005, thresholds=thresholds) eff_baseline = get_efficiencies(proba_baseline, eta, bins_number=30, errors=True, ignored_sideband=0.005, thresholds=thresholds_baseline) for i in thresholds: eff[i] = (eff[i][0], 100. * eff[i][1], 100. * eff[i][2], eff[i][3]) for i in thresholds_baseline: eff_baseline[i] = (eff_baseline[i][0], 100. * eff_baseline[i][1], 100. * eff_baseline[i][2], eff_baseline[i][3]) eff_total = OrderedDict() num = len(eff) + len(eff_baseline) for i in range(len(eff)): v = eff[eff.keys()[i]] v_baseline = eff_baseline[eff_baseline.keys()[i]] eff_total[num] = v eff_total[num - 1] = v_baseline num += -2 plot_fig = ErrorPlot(eff_total) plot_fig.ylim = (0, 100) plot_fig.plot(new_plot=True, figsize=(10, 7)) labels = [ 'Eff model = 20 %', 'Eff baseline = 20 %', 'Eff model = 50 %', 'Eff baseline = 50 %', 'Eff model = 80 %', 'Eff baseline = 80 %' ] plt.legend(labels, loc='best', prop={'size': 10}, framealpha=0.5, ncol=3) plt.xlabel(track_name + ' ' + particle_name + ' Pseudo Rapidity', size=15) plt.xticks(size=15) plt.ylabel('Efficiency / %', size=15) plt.yticks(size=15) plt.title('Flatness_SignalMVAEffVPseudoRapidity_' + track_name + ' ' + particle_name, size=15) if save_path != None: plt.savefig(save_path + "/" + 'Flatness_SignalMVAEffVPseudoRapidity_' + track_name + '_' + particle_name + ".png") if show == True: plt.show() plt.clf() plt.close()
def flatness_eta_figure(proba, proba_baseline, eta, track_name, particle_name, save_path=None, show=False): """ Plot signal efficiency vs pseudo rapidity figure. Parameters ---------- proba : array_like Predicted probabilities with array shape = [n_samples]. probas_baseline : array_like Baseline predicted probabilities with array shape = [n_samples]. eta : array_like Pseudo rapidity values with array shape = [n_samples]. track_name : string The track name. particle_name : string The particle name. save_path : string Path to a directory where the figure will saved. If None the figure will not be saved. show : boolean If true the figure will be displayed. """ thresholds = numpy.percentile(proba, 100 - numpy.array([20, 50, 80])) thresholds_baseline = numpy.percentile(proba_baseline, 100 - numpy.array([20, 50, 80])) eff = get_efficiencies(proba, eta, bins_number=30, errors=True, ignored_sideband=0.005, thresholds=thresholds) eff_baseline = get_efficiencies(proba_baseline, eta, bins_number=30, errors=True, ignored_sideband=0.005, thresholds=thresholds_baseline) for i in thresholds: eff[i] = (eff[i][0], 100. * eff[i][1], 100. * eff[i][2], eff[i][3]) for i in thresholds_baseline: eff_baseline[i] = (eff_baseline[i][0], 100. * eff_baseline[i][1], 100. * eff_baseline[i][2], eff_baseline[i][3]) eff_total = OrderedDict() num = len(eff) + len(eff_baseline) for i in range(len(eff)): v = eff[eff.keys()[i]] v_baseline = eff_baseline[eff_baseline.keys()[i]] eff_total[num] = v eff_total[num - 1] = v_baseline num += -2 plot_fig = ErrorPlot(eff_total) plot_fig.ylim = (0, 100) plot_fig.plot(new_plot=True, figsize=(10,7)) labels = ['Eff model = 20 %', 'Eff baseline = 20 %', 'Eff model = 50 %', 'Eff baseline = 50 %', 'Eff model = 80 %', 'Eff baseline = 80 %'] plt.legend(labels, loc='best',prop={'size':10}, framealpha=0.5, ncol=3) plt.xlabel(track_name + ' ' + particle_name + ' Pseudo Rapidity', size=15) plt.xticks(size=15) plt.ylabel('Efficiency / %', size=15) plt.yticks(size=15) plt.title('Flatness_SignalMVAEffVPseudoRapidity_' + track_name + ' ' + particle_name, size=15) if save_path != None: plt.savefig(save_path + "/" + 'Flatness_SignalMVAEffVPseudoRapidity_' + track_name + '_' + particle_name + ".png") if show == True: plt.show() plt.clf() plt.close()