def test_with_gauss_exp_example(calculator): mean, calculator = calculator() scan_values = np.linspace(1.15, 1.26, 50) poinull = POIarray(mean, scan_values) ci = ConfidenceInterval(calculator, poinull) interval = ci.interval() assert interval["lower"] == pytest.approx(1.1810371356602791, rel=0.1) assert interval["upper"] == pytest.approx(1.2156701172321935, rel=0.1) with pytest.raises(POIRangeError): poinull = POIarray( mean, scan_values[(scan_values >= 1.2) & (scan_values <= 1.205)]) ci = ConfidenceInterval(calculator, poinull) ci.interval() with pytest.raises(POIRangeError): poinull = POIarray(mean, scan_values[(scan_values >= 1.2)]) ci = ConfidenceInterval(calculator, poinull) ci.interval() with pytest.raises(POIRangeError): poinull = POIarray(mean, scan_values[(scan_values <= 1.205)]) ci = ConfidenceInterval(calculator, poinull) ci.interval()
def test_with_gauss_exp_example(calculator): Nsig, calculator = calculator() poinull = POIarray(Nsig, np.linspace(0.0, 25, 15)) poialt = POI(Nsig, 0) ul = UpperLimit(calculator, poinull, poialt) ul_qtilde = UpperLimit(calculator, poinull, poialt, qtilde=True) limits = ul.upperlimit(alpha=0.05, CLs=True) assert limits["observed"] == pytest.approx(15.725784747406346, rel=0.05) assert limits["expected"] == pytest.approx(11.464238503550177, rel=0.05) assert limits["expected_p1"] == pytest.approx(16.729552184042365, rel=0.1) assert limits["expected_p2"] == pytest.approx(23.718823517614066, rel=0.15) assert limits["expected_m1"] == pytest.approx(7.977175378979202, rel=0.1) assert limits["expected_m2"] == pytest.approx(5.805298972983304, rel=0.15) ul.upperlimit(alpha=0.05, CLs=False) ul_qtilde.upperlimit(alpha=0.05, CLs=True) # test error when scan range is too small with pytest.raises(POIRangeError): poinull = POIarray(Nsig, poinull.values[:5]) ul = UpperLimit(calculator, poinull, poialt) ul.upperlimit(alpha=0.05, CLs=True)
def test_with_gauss_qtilde(n, min_x): sigma_x = 0.032 minimizer = Minuit() bounds = (-10, 10) obs = zfit.Space("x", limits=bounds) mean = zfit.Parameter("mean", n * sigma_x) sigma = zfit.Parameter("sigma", 1.0) model = zfit.pdf.Gauss(obs=obs, mu=mean, sigma=sigma) data = model.sample(n=1000) nll = UnbinnedNLL(model=model, data=data) minimum = minimizer.minimize(loss=nll) minimum.hesse() x = minimum.params[mean]["value"] x_err = minimum.params[mean]["minuit_hesse"]["error"] x_min = x - x_err * 3 x_max = x + x_err * 3 x_min = max([x_min, min_x]) poinull = POIarray(mean, np.linspace(x_min, x_max, 50)) calculator = AsymptoticCalculator(nll, minimizer) ci = ConfidenceInterval(calculator, poinull, qtilde=True) ci.interval(alpha=0.05, printlevel=1)
def test_with_gauss_fluctuations(): x_true = -2.0 minimizer = Minuit() bounds = (-10, 10) obs = zfit.Space("x", limits=bounds) mean = zfit.Parameter("mean", 0) sigma = zfit.Parameter("sigma", 1.0) model = zfit.pdf.Gauss(obs=obs, mu=mean, sigma=sigma) npzfile = f"{notebooks_dir}/toys/FC_toys_{x_true}.npz" data = zfit.data.Data.from_numpy(obs=obs, array=np.load(npzfile)["x"]) nll = UnbinnedNLL(model=model, data=data) minimum = minimizer.minimize(loss=nll) minimum.hesse() toys_fname = f"{notebooks_dir}/toys/FC_toys_{x_true}.yml" calculator = FrequentistCalculator.from_yaml(toys_fname, minimum, minimizer) keys = np.unique([k[0].value for k in calculator.keys()]) keys.sort() poinull = POIarray(mean, keys) ci = ConfidenceInterval(calculator, poinull, qtilde=False) with pytest.warns(UserWarning): ci.interval(alpha=0.05, printlevel=0) ci = ConfidenceInterval(calculator, poinull, qtilde=True) ci.interval(alpha=0.05, printlevel=0)
def create_loss(): bounds = (0.1, 3.0) obs = zfit.Space("x", limits=bounds) # Data and signal np.random.seed(0) tau = -2.0 beta = -1 / tau bkg = np.random.exponential(beta, 300) peak = np.random.normal(1.2, 0.1, 25) data = np.concatenate((bkg, peak)) data = data[(data > bounds[0]) & (data < bounds[1])] N = len(data) data = zfit.data.Data.from_numpy(obs=obs, array=data) lambda_ = zfit.Parameter("lambda", -2.0, -4.0, -1.0) Nsig = zfit.Parameter("Nsig", 20.0, -20.0, N) Nbkg = zfit.Parameter("Nbkg", N, 0.0, N * 1.1) signal = zfit.pdf.Gauss(obs=obs, mu=1.2, sigma=0.1).create_extended(Nsig) background = zfit.pdf.Exponential(obs=obs, lambda_=lambda_).create_extended(Nbkg) tot_model = zfit.pdf.SumPDF([signal, background]) loss = ExtendedUnbinnedNLL(model=tot_model, data=data) poigen = POI(Nsig, 0.0) poieval = POIarray(Nsig, [0.0]) return loss, (Nsig, poigen, poieval)
def test_attributes(): loss, (mean, sigma) = create_loss() calculator = BaseCalculator(loss, Minuit()) poimean_1 = POIarray(mean, [1.0, 1.1, 1.2, 1.3]) poimean_2 = POI(mean, 1.2) test = BaseTest(calculator, poimean_1, poimean_2) assert test.poinull == poimean_1 assert test.poialt == poimean_2 assert test.calculator == calculator
def test_pois(): mean = zfit.Parameter("mu", 1.2, 0.1, 2) p0 = POI(mean, 0) p1 = POI(mean, 1.0) values = np.linspace(0.0, 1.0, 10) pn = POIarray(mean, values) pnc = POIarray(mean, values) for cls in [POI, POIarray]: with pytest.raises(ValueError): cls("mean", 0) with pytest.raises(TypeError): cls(mean) with pytest.raises(TypeError): POI(mean, values) with pytest.raises(TypeError): POIarray(mean, 0) repr(p0) repr(pn) assert p0.value == 0 assert p0.name == mean.name assert p0 != p1 assert all(pn.values == values) assert pn.name == mean.name assert len(pn) == len(values) iter(pn) assert pn == pnc assert hash(pn) == hash(pnc) assert pn != p0 assert pn != p1 assert pn[0] == p0 assert pn[1] != p0 assert pn[-1] == p1 pn1 = pn.append(12) assert pn1.values[-1] == 12 assert all(pn.values == values) assert pn1 != pn pn2 = pn.append([15, 20, 30]) assert pn2.values[-1] == 30 assert pn2.values[-2] == 20 assert pn2.values[-3] == 15 assert pn2 != pn {p0: "p0", p1: "p1", pn: "pn"}
def test_constructor(): with pytest.raises(TypeError): BaseTest() loss, (mean, sigma) = create_loss() calculator = BaseCalculator(loss, Minuit()) poimean = POIarray(mean, [1.0, 1.1, 1.2, 1.3]) poisigma = POI(sigma, 0.1) with pytest.raises(TypeError): BaseTest(calculator) with pytest.raises(TypeError): BaseTest(calculator, poimean, [poisigma]) with pytest.raises(TypeError): BaseTest("calculator", poimean, poisigma)
def test_asymptotic_calculator_one_poi(): with pytest.raises(TypeError): AsymptoticCalculator() loss, (mean, sigma) = create_loss() calc = AsymptoticCalculator(loss, Minuit()) poi_null = POIarray(mean, [1.15, 1.2, 1.25]) poi_alt = POI(mean, 1.2) dataset = calc.asimov_dataset(poi_alt) assert all(is_valid_data(d) for d in dataset) loss = calc.asimov_loss(poi_alt) assert is_valid_loss(loss) null_nll = calc.asimov_nll(pois=poi_null, poialt=poi_alt) assert null_nll[0] >= null_nll[1] assert null_nll[2] >= null_nll[1]
def test_base_calculator(calculator): with pytest.raises(TypeError): calculator() loss, (mean, sigma) = create_loss() with pytest.raises(ValueError): calculator("loss", Minuit()) with pytest.raises(ValueError): calculator(loss, "Minuit()") calc_loss = calculator(loss, Minuit()) with pytest.raises(ValueError): calc_loss.bestfit = "bestfit" bestfit = calc_loss.bestfit calc_fitresult = calculator(bestfit, calc_loss.minimizer) assert calc_loss.bestfit == calc_fitresult.bestfit assert calc_loss.loss == calc_fitresult.loss mean_poi = POIarray(mean, [1.15, 1.2, 1.25]) mean_nll = calc_loss.obs_nll(pois=mean_poi) calc_loss.obs_nll(pois=mean_poi) # get from cache assert mean_nll[0] >= mean_nll[1] assert mean_nll[2] >= mean_nll[1] assert calc_loss.obs_nll(mean_poi[0]) == mean_nll[0] assert calc_loss.obs_nll(mean_poi[1]) == mean_nll[1] assert calc_loss.obs_nll(mean_poi[2]) == mean_nll[2] mean_poialt = POI(mean, 1.2) pvalue = lambda: calc_loss.pvalue(poinull=mean_poi, poialt=mean_poialt) exp_pvalue = lambda: calc_loss.expected_pvalue( poinull=mean_poi, poialt=mean_poialt, nsigma=np.arange(-2, 3, 1) ) exp_poi = lambda: calc_loss.expected_poi( poinull=mean_poi, poialt=mean_poialt, nsigma=np.arange(-2, 3, 1) ) if calculator == BaseCalculator: with pytest.raises(NotImplementedError): pvalue() with pytest.raises(NotImplementedError): exp_pvalue() else: pvalue() exp_pvalue() model = calc_loss.model[0] sampler = model.create_sampler(n=10000) assert is_valid_data(sampler) loss = calc_loss.lossbuilder(model=[model], data=[sampler], weights=None) assert is_valid_loss(loss) with pytest.raises(ValueError): calc_loss.lossbuilder(model=[model, model], data=[sampler]) with pytest.raises(ValueError): calc_loss.lossbuilder(model=[model], data=[sampler, calc_loss.data[0]]) with pytest.raises(ValueError): calc_loss.lossbuilder(model=[model], data=[sampler], weights=[]) with pytest.raises(ValueError): calc_loss.lossbuilder( model=[model], data=[sampler], weights=[np.ones(10000), np.ones(10000)] ) assert calc_loss.get_parameter(mean_poi.name) == mean with pytest.raises(KeyError): calc_loss.get_parameter("dummy_parameter")