def analyse_coverage(toys_s_b, all_mu_sig, **kwargs):
coverage_CLs, coverage_CLs_b = [], []
for mu_sig in all_mu_sig:
cov_CLs, cov_CLs_b = get_coverage(toys_s_b, all_mu_sig, mu_sig,
**kwargs)
coverage_CLs.append(cov_CLs)
coverage_CLs_b.append(cov_CLs_b)
msg.info(
"analyse_coverage",
"mu_sig = {0:.4f} with Coverage(CLs) = {1:.4f} and Coverage(CL_s+b) = {2:.4f}"
.format(mu_sig, cov_CLs, cov_CLs_b),
verbose_level=1)
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111)
ax.plot(all_mu_sig, coverage_CLs_b, '--', c='k', label=r"${\rm CL}_{s+b}$")
ax.plot(all_mu_sig, coverage_CLs, '-', c='r', label=r"${\rm CL}_{s}$")
ax.set_xlim(all_mu_sig[0], all_mu_sig[-1])
ax.set_ylim(0.8, 1.05)
plt.ylabel("Expected coverage")
plt.xlabel(r"$\mu_{\rm sig}^{\rm true}$")
plt.title(kwargs.get("title", "CL < 95%% interval"))
plt.legend(loc="lower left")
plt.show()
if type(plotting.document) is not None:
fig.savefig(plotting.document, format='pdf')
plt.close()
def set_save_file ( fname_ ) :
close_save_file()
global document
if type(fname_) is str :
if fname_[-4:] != ".pdf" : fname_ = fname_ + ".pdf"
msg.info("Plotting.plotting.set_save_file", "Opening pdf file {0}".format(fname_), verbose_level=0)
document = PdfPages(fname_)
else :
msg.error("Plotting.plotting.set_save_file", "Filename must be a str")
def get_upper_limits(toys_s_b, all_mu_sig, mu_sig, **kwargs):
msg.info("get_upper_limits",
"Getting upper limits for mu_sig = {0:.4f}".format(mu_sig),
verbose_level=1)
upp_lim_CLs, upp_lim_CLs_b = [], []
for q in toys_s_b[mu_sig]:
lim_CLs, lim_CLs_b = 0., 0.
CL_b = get_CL(q, toys_s_b[0])
for test_mu in all_mu_sig:
CL_s_b = get_CL(q, toys_s_b[test_mu])
if CL_b == 0: CL_s = 1
else: CL_s = CL_s_b / CL_b
if CL_s_b > 0.05: lim_CLs_b = test_mu
if CL_s > 0.05: lim_CLs = test_mu
upp_lim_CLs.append(lim_CLs)
upp_lim_CLs_b.append(lim_CLs_b)
return upp_lim_CLs, upp_lim_CLs_b
def throw_and_fit_toys(model, n_toys, **kwargs):
set_fit_model(model)
k1_true, k2_true = model.coefficients[1], model.coefficients[2]
toys_q, toys_dTNLL, toys_k1, toys_k2 = [], [], [], []
for i in range(n_toys):
if n_toys > 150 and (10 * i) % n_toys == 0:
msg.info("Confidence.DM_fitting_2.throw_and_fit_toys",
"Processing toy {0} out of {1}".format(i, n_toys),
verbose_level=1)
n_tries = 0
while n_tries > -1:
if n_tries == 1000:
msg.fatal("Confidence.DM_fitting_2.throw_and_fit_toys",
"Toy does not converge after 1000 attempts")
try:
y, ey = model.throw_toy(fabs=True)
set_data(y)
tnll_s_b, tnll_s_b_true, k1, k1_err_lo, k1_err_hi, k2, k2_err_lo, k2_err_hi = get_fitted_TNLL_and_kappas(
model.coefficients, (0.1, 0.1, 0.1), (False, False, False),
k_true=(k1_true, k2_true))
tnll_b = get_fitted_TNLL((1., 1., 0.), (0.1, 0.1, 0.1),
(False, True, True))
q = TNLL_to_likelihood(tnll_s_b) / TNLL_to_likelihood(tnll_b)
dTNLL = tnll_s_b_true - tnll_s_b
n_tries = -1
except Exception as e:
msg.warning(
"Confidence.DM_fitting_2.throw_and_fit_toys",
"Error {0} occurred when fitting toy. Retrying.".format(e))
n_tries = n_tries + 1
toys_q.append(float(q))
toys_dTNLL.append(float(dTNLL))
toys_k1.append((k1, k1_err_lo, k1_err_hi))
toys_k2.append((k2, k2_err_lo, k2_err_hi))
#print( "{0} {1} with measurements {2} +/- {3} and {4} +/- {5}".format(k1_true, k2_true, k1, k1_err, k2, k2_err) )
toys_q.sort()
toys_dTNLL.sort()
return toys_q, toys_dTNLL, toys_k1, toys_k2
def get_q_distribution_for_kappa(model, k1, k2, **kwargs):
force_new_toys = kwargs.get("force_new_toys", True)
tag = kwargs.get("tag", "untagged")
n_bins = kwargs.get("n_bins", -1)
n_toys = kwargs.get("n_toys", -1)
x_low = kwargs.get("x_low", -1.)
x_high = kwargs.get("x_high", -1.)
rnd_seed = kwargs.get("rnd_seed", -1)
plot_pulls = kwargs.get("plot_pulls", False)
toys_q, toys_dTNLL, toys_k1, toys_k2 = [], [], [], []
is_NULL_hypothesis = (k1 == 1 and k2 == 0)
if is_NULL_hypothesis is True:
label = tag + "(NULL)"
fname_toys = "tmp/{0}_toys_b.dat".format(tag)
else:
label = tag + "(k1 = {0:.2f}, k2 = {1:.2f})".format(k1, k2)
fname_toys = "tmp/{0}_toys_s_{1:.2f}_{2:.2f}.dat".format(tag, k1, k2)
if force_new_toys is False:
toys_opened_success, toys_q, toys_dTNLL, toys_k1, toys_k2 = open_toys_file(
fname_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
rnd_seed=rnd_seed)
force_new_toys = not toys_opened_success
if force_new_toys is True:
msg.info(
"study_WilksTheorem.py",
"Throwing {0} toys around the {1} hypothesis".format(
n_toys, label))
model.coefficients[1] = k1
model.coefficients[2] = k2
toys_q, toys_dTNLL, toys_k1, toys_k2 = fit.throw_and_fit_toys(
model, n_toys)
msg.info(
"study_WilksTheorem.py",
"Saving {0} hypothesis toys to file {1}".format(label, fname_toys))
save_toys_file(toys_q,
toys_dTNLL,
toys_k1,
toys_k2,
fname_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
rnd_seed=rnd_seed)
if False in [
len(x) == n_toys for x in [toys_q, toys_dTNLL, toys_k1, toys_k2]
]:
msg.fatal(
"study_WilksTheorem.py",
"Failure loading toys for the {1} hypothesis".format(
n_toys, label))
else:
msg.info(
"study_WilksTheorem.py",
"{0} toys successfully loaded for the {1} hypothesis".format(
n_toys, label))
return toys_q, toys_dTNLL, toys_k1, toys_k2
def get_q_distribution_for_signal_strength(model, correlation, mu_sig,
**kwargs):
force_new_toys = kwargs.get("force_new_toys", True)
tag = kwargs.get("tag", "untagged")
n_bins = kwargs.get("n_bins", -1)
n_toys = kwargs.get("n_toys", -1)
x_low = kwargs.get("x_low", -1.)
x_high = kwargs.get("x_high", -1.)
rnd_seed = kwargs.get("rnd_seed", -1)
plot_pulls = kwargs.get("plot_pulls", False)
toys = []
is_NULL_hypothesis = mu_sig == 0
if is_NULL_hypothesis is True:
label = tag + "(NULL)"
fname_toys = "tmp/{0}_toys_b.dat".format(tag)
else:
label = tag + "(mu_sig = {0:.5f})".format(mu_sig)
fname_toys = "tmp/{0}_toys_s_{1:.5f}.dat".format(tag, mu_sig)
if force_new_toys is False:
toys_opened_success, toys = open_toys_file(fname_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
rnd_seed=rnd_seed)
force_new_toys = not toys_opened_success
if force_new_toys is True:
msg.info(
"study_CLs.py",
"Throwing {0} toys around the {1} hypothesis".format(
n_toys, label))
model.coefficients[1] = mu_sig
toys = fit.throw_and_fit_toys(
model,
correlation,
n_toys,
plot_pulls=plot_pulls,
pulls_xlabel=
r"$\frac{\hat{\mu}_{\rm sig} - \mu^{\rm true}_{\rm sig}}{\hat{\sigma}_{\rm sig}}$",
pulls_label=r"Toys with $\mu^{\rm true}_{\rm sig} = " +
"{0:.1f}$".format(mu_sig))
msg.info(
"study_CLs.py",
"Saving {0} hypothesis toys to file {1}".format(label, fname_toys))
save_toys_file(toys,
fname_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
rnd_seed=rnd_seed)
if len(toys) == n_toys:
msg.info(
"study_CLs.py",
"{0} toys successfully loaded for the {1} hypothesis".format(
n_toys, label))
return toys
def test():
plotting.set_style()
msg.VERBOSE_LEVEL = 1
plotting.set_save_file("out/test.pdf")
#
# Config constants
#
n_bins = 11
n_toys = 4000
x_low, x_high = 5, 15
rnd_seed = 100
plot_asimovs = True
plot_pulls = False
plot_q = False
plot_a_toy = True
plot_CL_for_median_toys = False
force_new_toys = False
mu_sig_interval = 0.2
mu_sig_npoints = 40
do_analyse_coverage = False
do_analyse_sensitivities = False
#
# Create sig + bkg expectations for moderate and large backgrounds, and plot them
#
model1 = DM.create_simple_DM_model_1(scale_bkg=0.8,
x_low=x_low,
x_high=x_high,
n_bins=n_bins)
model2 = DM.create_simple_DM_model_1(scale_bkg=5.,
x_low=x_low,
x_high=x_high,
n_bins=n_bins)
if plot_asimovs is True:
model1.plot_asimov(
x_min=5.,
x_max=15.,
y_min=0,
y_max=50,
xlabel="Observable",
ylabel="Events / bin width",
labels=("SM bkg (moderate)",
r"DM signal ($\mu_{\rm sig}^{\rm true}=1$)"),
title=r"Model1")
model2.plot_asimov(
x_min=5.,
x_max=15.,
y_min=0,
y_max=260,
xlabel="Observable",
ylabel="Events / bin width",
labels=("SM bkg (large)",
r"DM signal ($\mu_{\rm sig}^{\rm true}=1$)"),
title=r"Model2")
#
# Create an experimental correlation between the fitted bins
#
utils.set_numpy_random_seed(rnd_seed)
correlation = generate_correlation_matrix(n_bins, randomise=False)
#
# Plot a toy if desired
#
if plot_a_toy is True:
model1.plot_toy(x_min=5.,
x_max=15.,
y_max=50,
xlabel="Observable",
ylabel="Events / bin width",
labels=("SM bkg (moderate)",
r"DM signal ($\mu_{\rm sig}^{\rm true}=1$)"),
title=r"Model1")
model2.plot_toy(x_min=5.,
x_max=15.,
y_max=260,
xlabel="Observable",
ylabel="Events / bin width",
labels=("SM bkg (large)",
r"DM signal ($\mu_{\rm sig}^{\rm true}=1$)"),
title=r"Model2")
utils.set_numpy_random_seed(rnd_seed)
#
# Evaluate the expected significance for the two models
#
expected_significance_model1 = fit.get_expected_mu_sig_error(
model1, correlation)
expected_significance_model2 = fit.get_expected_mu_sig_error(
model2, correlation)
msg.info(
"study_CLs.py", "Expected significance of model 1 is {0}".format(
1. / expected_significance_model1))
msg.info(
"study_CLs.py", "Expected significance of model 2 is {0}".format(
1. / expected_significance_model2))
#
# Load toy distribution of q for the background only case, or throw+fit them if needed (MODEL1)
#
toys_NULL = get_q_distribution_for_signal_strength(
model1,
correlation,
0,
tag="model1",
force_new_toys=force_new_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
plot_pulls=plot_pulls)
if plot_q is True:
plotting.plot_histo_from_data(
400, [toys_NULL],
logy=True,
xmax=40,
xlabel=r"Test statistic: $q = \frac{L(s+b)}{L(b)}$",
ylabel="Num. toys",
labels=["Model1, NULL hypothesis"])
#
# Load toy distribution of q for the signal cases, or throw+fit them if needed (MODEL1)
#
toys_model1_s = {}
toys_model1_s[0] = toys_NULL
signal_points = [0]
for i in range(mu_sig_npoints):
mu_sig = (1 + i) * mu_sig_interval
signal_points.append(mu_sig)
toys_model1_s[mu_sig] = get_q_distribution_for_signal_strength(
model1,
correlation,
mu_sig,
tag="model1",
force_new_toys=force_new_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
plot_pulls=plot_pulls)
if plot_q is False: continue
plotting.plot_histo_from_data(
400, [toys_model1_s[mu_sig]],
logy=True,
xmax=40,
xlabel=r"Test statistic: $q = \frac{L(s+b)}{L(b)}$",
ylabel="Num. toys",
labels=["Model1, $\mu_{{sig}}={0:.5f}$".format(mu_sig)])
#
# Look at CLs and CLs+b coverage at each mu (MODEL1)
#
if do_analyse_coverage is True:
if plot_CL_for_median_toys is True:
for mu_sig in signal_points:
plot_CL_profile(
toys_model1_s,
signal_points,
mu_sig,
int(n_toys / 2),
title=
"Model1: toy with median $q_{{\\rm obs}}$, $\mu_{{\\rm sig}}^{{\\rm true}} = {0:.1f}$"
.format(mu_sig))
analyse_coverage(toys_model1_s,
signal_points,
title=r"Model1: CL < 95% interval")
#
# Get distribution of toys for model2
#
toys_model2_NULL = get_q_distribution_for_signal_strength(
model2,
correlation,
0,
tag="model2",
force_new_toys=force_new_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
plot_pulls=plot_pulls)
toys_model2_s = {}
toys_model2_s[0] = toys_model2_NULL
for mu in signal_points[1:]:
toys_model2_s[mu] = get_q_distribution_for_signal_strength(
model2,
correlation,
mu,
tag="model2",
force_new_toys=force_new_toys,
n_bins=n_bins,
n_toys=n_toys,
x_low=x_low,
x_high=x_high,
plot_pulls=plot_pulls)
#
# Look at CLs and CLs+b coverage at each mu (MODEL2)
#
if do_analyse_coverage is True:
if plot_CL_for_median_toys is True:
for mu_sig in signal_points:
plot_CL_profile(
toys_model2_s,
signal_points,
mu_sig,
int(n_toys / 2),
title=
"Model1: toy with median $q_{{\\rm obs}}$, $\mu_{{\\rm sig}}^{{\\rm true}} = {0:.1f}$"
.format(mu_sig))
analyse_coverage(toys_model2_s,
signal_points,
title=r"Model2: CL < 95% interval")
#
# Look at comparitive sensitivities
#
if do_analyse_sensitivities is True:
analyse_sensitivities(toys_model1_s, toys_model2_s, signal_points)
# Clean up
plotting.close_save_file()
def throw_and_fit_toys(model, correlation, n_toys, **kwargs):
set_fit_model(model)
mu_true = model.coefficients[1]
toys_q = []
pulls_mu_sig = []
for i in range(n_toys):
if i > 0 and (10 * i) % n_toys == 0:
msg.info("Confidence.DM_fitting.throw_and_fit_toys",
"Processing toy {0} out of {1}".format(i, n_toys),
verbose_level=1)
n_tries = 0
while n_tries > -1:
if n_tries == 1000:
msg.fatal(
"Confidence.DM_fitting.throw_and_fit_toys",
"Covariance matrix is still singular after 1000 attempts")
try:
y, ey = model.throw_toy()
set_data_and_covariance(y, ey, correlation)
n_tries = -1
except Exception as e:
msg.warning(
"Confidence.DM_fitting.throw_and_fit_toys",
"Error {0} occurred when throwing toy. Retrying.".format(
e),
verbose_level=1)
n_tries = n_tries + 1
tnll_s_b, mu_sig, mu_sig_err = get_fitted_TNLL_and_signal_strength(
model.coefficients, (0.5, 0.5), (False, False))
if kwargs.get("plot_first_profile", False) is True:
x, y = [], []
for i in range(21):
this_x = mu_sig + (i * 0.2 - 2.0) * mu_sig_err
x.append(this_x)
y.append(
get_fitted_TNLL((1.0, this_x), (0.5, 0.5), (False, True)))
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111)
ax.plot(x, y, "-", c="r")
plt.show()
if type(plotting.document) is not None:
fig.savefig(plotting.document, format='pdf')
plt.close(fig)
tnll_b = get_fitted_TNLL((1., 0.), (0.5, 0.5), (False, True))
q = TNLL_to_likelihood(tnll_s_b) / TNLL_to_likelihood(tnll_b)
toys_q.append(float(q))
pulls_mu_sig.append((mu_sig - mu_true) / mu_sig_err)
toys_q.sort()
mean, sem, sd, se = stats.get_mean_sem_sd_se(pulls_mu_sig)
msg.info("Confidence.DM_fitting.throw_and_fit_toys",
" Pulls mean = {0:.4f} +/- {1:.4f}".format(mean, sem))
msg.info("Confidence.DM_fitting.throw_and_fit_toys",
" Pulls sd = {0:.4f} +/- {1:.4f}".format(sd, se))
if kwargs.get("plot_pulls", False) is False:
return toys_q
msg.info("Confidence.DM_fitting.throw_and_fit_toys",
"Plotting pulls for mu_true = {0:.5f}".format(mu_true))
plotting.plot_histo_from_data(400, [pulls_mu_sig],
xmin=-5,
xmax=5,
ymin=0.,
xlabel=kwargs.get("pulls_xlabel", "Pull"),
ylabel="Num. toys",
labels=[kwargs.get("pulls_label", "")],
overlay_gauss=True)
return toys_q