def test_pois():
mean = zfit.Parameter("mu", 1.2, 0.1, 2)
p0 = POI(mean, 0)
p1 = POI(mean, 1.0)
values = np.linspace(0.0, 1.0, 10)
pn = POIarray(mean, values)
pnc = POIarray(mean, values)
for cls in [POI, POIarray]:
with pytest.raises(ValueError):
cls("mean", 0)
with pytest.raises(TypeError):
cls(mean)
with pytest.raises(TypeError):
POI(mean, values)
with pytest.raises(TypeError):
POIarray(mean, 0)
repr(p0)
repr(pn)
assert p0.value == 0
assert p0.name == mean.name
assert p0 != p1
assert all(pn.values == values)
assert pn.name == mean.name
assert len(pn) == len(values)
iter(pn)
assert pn == pnc
assert hash(pn) == hash(pnc)
assert pn != p0
assert pn != p1
assert pn[0] == p0
assert pn[1] != p0
assert pn[-1] == p1
pn1 = pn.append(12)
assert pn1.values[-1] == 12
assert all(pn.values == values)
assert pn1 != pn
pn2 = pn.append([15, 20, 30])
assert pn2.values[-1] == 30
assert pn2.values[-2] == 20
assert pn2.values[-3] == 15
assert pn2 != pn
{p0: "p0", p1: "p1", pn: "pn"}
def test_with_gauss_exp_example(calculator):
Nsig, calculator = calculator()
poinull = POIarray(Nsig, np.linspace(0.0, 25, 15))
poialt = POI(Nsig, 0)
ul = UpperLimit(calculator, poinull, poialt)
ul_qtilde = UpperLimit(calculator, poinull, poialt, qtilde=True)
limits = ul.upperlimit(alpha=0.05, CLs=True)
assert limits["observed"] == pytest.approx(15.725784747406346, rel=0.05)
assert limits["expected"] == pytest.approx(11.464238503550177, rel=0.05)
assert limits["expected_p1"] == pytest.approx(16.729552184042365, rel=0.1)
assert limits["expected_p2"] == pytest.approx(23.718823517614066, rel=0.15)
assert limits["expected_m1"] == pytest.approx(7.977175378979202, rel=0.1)
assert limits["expected_m2"] == pytest.approx(5.805298972983304, rel=0.15)
ul.upperlimit(alpha=0.05, CLs=False)
ul_qtilde.upperlimit(alpha=0.05, CLs=True)
# test error when scan range is too small
with pytest.raises(POIRangeError):
poinull = POIarray(Nsig, poinull.values[:5])
ul = UpperLimit(calculator, poinull, poialt)
ul.upperlimit(alpha=0.05, CLs=True)
def create_loss():
bounds = (0.1, 3.0)
obs = zfit.Space("x", limits=bounds)
# Data and signal
np.random.seed(0)
tau = -2.0
beta = -1 / tau
bkg = np.random.exponential(beta, 300)
peak = np.random.normal(1.2, 0.1, 25)
data = np.concatenate((bkg, peak))
data = data[(data > bounds[0]) & (data < bounds[1])]
N = len(data)
data = zfit.data.Data.from_numpy(obs=obs, array=data)
lambda_ = zfit.Parameter("lambda", -2.0, -4.0, -1.0)
Nsig = zfit.Parameter("Nsig", 20.0, -20.0, N)
Nbkg = zfit.Parameter("Nbkg", N, 0.0, N * 1.1)
signal = zfit.pdf.Gauss(obs=obs, mu=1.2, sigma=0.1).create_extended(Nsig)
background = zfit.pdf.Exponential(obs=obs, lambda_=lambda_).create_extended(Nbkg)
tot_model = zfit.pdf.SumPDF([signal, background])
loss = ExtendedUnbinnedNLL(model=tot_model, data=data)
poigen = POI(Nsig, 0.0)
poieval = POIarray(Nsig, [0.0])
return loss, (Nsig, poigen, poieval)
def test_constructor():
with pytest.raises(TypeError):
ConfidenceInterval()
loss, mean = create_loss()
calculator = BaseCalculator(loss, Minuit())
poi_1 = POI(mean, 1.5)
poi_2 = POI(mean, 1.2)
with pytest.raises(TypeError):
ConfidenceInterval(calculator)
with pytest.raises(TypeError):
ConfidenceInterval(calculator, [poi_1], poi_2, qtilde=True)
with pytest.raises(TypeError):
ConfidenceInterval(calculator, [poi_1], [poi_2], qtilde=False)
def test_constructor():
with pytest.raises(TypeError):
UpperLimit()
loss, (Nsig, Nbkg) = create_loss()
calculator = BaseCalculator(loss, Minuit())
poi_1 = POI(Nsig, 0.0)
poi_2 = POI(Nsig, 2.0)
with pytest.raises(TypeError):
UpperLimit(calculator)
with pytest.raises(TypeError):
UpperLimit(calculator, poi_1)
with pytest.raises(TypeError):
UpperLimit(calculator, [poi_1], poi_2)
def test_attributes():
loss, (mean, sigma) = create_loss()
calculator = BaseCalculator(loss, Minuit())
poimean_1 = POIarray(mean, [1.0, 1.1, 1.2, 1.3])
poimean_2 = POI(mean, 1.2)
test = BaseTest(calculator, poimean_1, poimean_2)
assert test.poinull == poimean_1
assert test.poialt == poimean_2
assert test.calculator == calculator
def test_with_asymptotic_calculator():
loss, (Nsig, Nbkg) = create_loss()
calculator = AsymptoticCalculator(loss, Minuit())
poinull = POI(Nsig, 0)
discovery_test = Discovery(calculator, poinull)
pnull, significance = discovery_test.result()
assert pnull == pytest.approx(0.0007571045089567185, abs=0.05)
assert significance == pytest.approx(3.1719464953752565, abs=0.05)
assert significance >= 3
def test_with_frequentist_calculator():
loss, (Nsig, Nbkg) = create_loss()
calculator = FrequentistCalculator.from_yaml(
f"{notebooks_dir}/toys/discovery_freq_zfit_toys.yml", loss, Minuit())
poinull = POI(Nsig, 0)
discovery_test = Discovery(calculator, poinull)
pnull, significance = discovery_test.result()
assert pnull == pytest.approx(0.0004, rel=0.05, abs=0.0005)
assert significance == pytest.approx(3.3527947805048592, rel=0.05, abs=0.1)
assert significance >= 3
def test_counting_with_frequentist_calculator():
(
loss,
Nsig,
) = create_loss_counting()
calculator = FrequentistCalculator(loss, Minuit(), ntoysnull=1000)
poinull = POI(Nsig, 0)
discovery_test = Discovery(calculator, poinull)
pnull, significance = discovery_test.result()
assert significance < 2
def test_counting_with_asymptotic_calculator():
(
loss,
Nsig,
) = create_loss_counting()
calculator = AsymptoticCalculator(loss, Minuit())
poinull = POI(Nsig, 0)
discovery_test = Discovery(calculator, poinull)
pnull, significance = discovery_test.result()
assert significance < 2
def test_constructor():
with pytest.raises(TypeError):
BaseTest()
loss, (mean, sigma) = create_loss()
calculator = BaseCalculator(loss, Minuit())
poimean = POIarray(mean, [1.0, 1.1, 1.2, 1.3])
poisigma = POI(sigma, 0.1)
with pytest.raises(TypeError):
BaseTest(calculator)
with pytest.raises(TypeError):
BaseTest(calculator, poimean, [poisigma])
with pytest.raises(TypeError):
BaseTest("calculator", poimean, poisigma)
def test_asymptotic_calculator_one_poi():
with pytest.raises(TypeError):
AsymptoticCalculator()
loss, (mean, sigma) = create_loss()
calc = AsymptoticCalculator(loss, Minuit())
poi_null = POIarray(mean, [1.15, 1.2, 1.25])
poi_alt = POI(mean, 1.2)
dataset = calc.asimov_dataset(poi_alt)
assert all(is_valid_data(d) for d in dataset)
loss = calc.asimov_loss(poi_alt)
assert is_valid_loss(loss)
null_nll = calc.asimov_nll(pois=poi_null, poialt=poi_alt)
assert null_nll[0] >= null_nll[1]
assert null_nll[2] >= null_nll[1]
def test_base_calculator(calculator):
with pytest.raises(TypeError):
calculator()
loss, (mean, sigma) = create_loss()
with pytest.raises(ValueError):
calculator("loss", Minuit())
with pytest.raises(ValueError):
calculator(loss, "Minuit()")
calc_loss = calculator(loss, Minuit())
with pytest.raises(ValueError):
calc_loss.bestfit = "bestfit"
bestfit = calc_loss.bestfit
calc_fitresult = calculator(bestfit, calc_loss.minimizer)
assert calc_loss.bestfit == calc_fitresult.bestfit
assert calc_loss.loss == calc_fitresult.loss
mean_poi = POIarray(mean, [1.15, 1.2, 1.25])
mean_nll = calc_loss.obs_nll(pois=mean_poi)
calc_loss.obs_nll(pois=mean_poi) # get from cache
assert mean_nll[0] >= mean_nll[1]
assert mean_nll[2] >= mean_nll[1]
assert calc_loss.obs_nll(mean_poi[0]) == mean_nll[0]
assert calc_loss.obs_nll(mean_poi[1]) == mean_nll[1]
assert calc_loss.obs_nll(mean_poi[2]) == mean_nll[2]
mean_poialt = POI(mean, 1.2)
pvalue = lambda: calc_loss.pvalue(poinull=mean_poi, poialt=mean_poialt)
exp_pvalue = lambda: calc_loss.expected_pvalue(
poinull=mean_poi, poialt=mean_poialt, nsigma=np.arange(-2, 3, 1)
)
exp_poi = lambda: calc_loss.expected_poi(
poinull=mean_poi, poialt=mean_poialt, nsigma=np.arange(-2, 3, 1)
)
if calculator == BaseCalculator:
with pytest.raises(NotImplementedError):
pvalue()
with pytest.raises(NotImplementedError):
exp_pvalue()
else:
pvalue()
exp_pvalue()
model = calc_loss.model[0]
sampler = model.create_sampler(n=10000)
assert is_valid_data(sampler)
loss = calc_loss.lossbuilder(model=[model], data=[sampler], weights=None)
assert is_valid_loss(loss)
with pytest.raises(ValueError):
calc_loss.lossbuilder(model=[model, model], data=[sampler])
with pytest.raises(ValueError):
calc_loss.lossbuilder(model=[model], data=[sampler, calc_loss.data[0]])
with pytest.raises(ValueError):
calc_loss.lossbuilder(model=[model], data=[sampler], weights=[])
with pytest.raises(ValueError):
calc_loss.lossbuilder(
model=[model], data=[sampler], weights=[np.ones(10000), np.ones(10000)]
)
assert calc_loss.get_parameter(mean_poi.name) == mean
with pytest.raises(KeyError):
calc_loss.get_parameter("dummy_parameter")
plot_pdf_data(data=rare_data,
model=model_rare,
title='$B^0 -> K^{*} (-> K^+ \pi^-) \mu^+ \mu^-$')
plot_pdf_data(data=reso_data,
model=model_reso,
title='$B^0 -> K^{*} (-> K^+ \pi^-) J/\psi (-> \mu^+ \mu^-)$')
plt.show()
# Hesse is not yet supported with weights
# result_simult.hesse() # error calculation using hesse
errors = result_simult.error(
[mu, rare_sig_yield,
reso_sig_yield]) # error calculation using minos, just for a few
# parameters as it is quite expensive
pprint(errors)
# ------------------------------------------------
# DETERMINING THE SIGNIFICANCE
# ------------------------------------------------
# using hepstats for the limits
from hepstats.hypotests import Discovery
from hepstats.hypotests.calculators import AsymptoticCalculator
from hepstats.hypotests.parameters import POI
calculator = AsymptoticCalculator(simultaneous_loss, minimizer)
poinull = POI(rare_sig_yield, 0)
discovery_test = Discovery(calculator, poinull)
pnull, significance = discovery_test.result()
print(f'pnull: {pnull} with significance {significance}')
