def test_gamma_ab_expected_loss_relative_ci(): modelA = GammaModel(shape=25, rate=10000) modelB = GammaModel(shape=30, rate=10000) abtest = GammaABTest(modelA, modelB, 1000000, 42) assert abtest.expected_loss_relative_ci( method="exact", variant="A") == approx( (-0.2302812912, 0.8907846796), rel=1e-8) assert abtest.expected_loss_relative_ci( method="MC", variant="A") == approx( (-0.2302812912, 0.8907846796), rel=1e-2) assert abtest.expected_loss_relative_ci( method="exact", variant="B") == approx( (-0.4711190487, 0.2991759050), rel=1e-8) assert abtest.expected_loss_relative_ci( method="MC", variant="B") == approx( (-0.4711190487, 0.2991759050), rel=1e-2) ci = abtest.expected_loss_relative_ci(method="exact", variant="all") assert ci[0] == approx([-0.2302812912, 0.8907846796], rel=1e-8) assert ci[1] == approx([-0.4711190487, 0.2991759050], rel=1e-8) ci = abtest.expected_loss_relative_ci(method="MC", variant="all") assert ci[0] == approx([-0.2302812912, 0.8907846796], rel=1e-2) assert ci[1] == approx([-0.4711190487, 0.2991759050], rel=1e-2)
def test_gamma_mv_expected_loss_relative(): modelA = GammaModel(shape=2500, rate=1000000) modelB = GammaModel(shape=3000, rate=1000000) abtest = GammaABTest(modelA, modelB, 1000000, 42) mvtest = GammaMVTest({"A": modelA, "B": modelB}, 1000000, 42) ab_result = abtest.expected_loss_relative(method="exact", variant="A") mv_result = mvtest.expected_loss_relative( method="exact", control="B", variant="A") assert ab_result == approx(mv_result, rel=1e-8) ab_result = abtest.expected_loss_relative(method="MC", variant="A") mv_result = mvtest.expected_loss_relative( method="MC", control="B", variant="A") assert ab_result == approx(mv_result, rel=1e-2) ab_result = abtest.expected_loss_relative(method="exact", variant="B") mv_result = mvtest.expected_loss_relative(method="exact", variant="B") assert ab_result == approx(mv_result, rel=1e-8) ab_result = abtest.expected_loss_relative(method="MC", variant="B") mv_result = mvtest.expected_loss_relative(method="MC", variant="B") assert ab_result == approx(mv_result, rel=1e-2)
def test_gamma_mv_check_method(): models = { "A": GammaModel(shape=25, rate=10000), "B": GammaModel(shape=30, rate=10000) } mvtest = GammaMVTest(models) with raises(ValueError): mvtest.probability(method="new")
def test_gamma_model_stats(): model = GammaModel(shape=3, rate=4) assert model.mean() == approx(0.75) assert model.var() == approx(0.1875) assert model.std() == approx(0.43301270189) assert model.pdf(0.1) == approx(0.2145024147) assert model.pdf(0) == 0 assert model.pdf(1) == approx(0.5861004444) assert model.cdf(0) == 0 assert model.cdf(1) == approx(0.7618966944) assert model.ppf(0.1) == approx(0.2755163) assert model.ppf(0.5) == approx(0.6685151)
def test_gamma_mv_expected_loss_relative_vs_all(): models = { "A": GammaModel(shape=25, rate=10000), "B": GammaModel(shape=30, rate=10000), "C": GammaModel(shape=40, rate=11000) } mvtest = GammaMVTest(models, 1000000, 42) assert mvtest.expected_loss_relative_vs_all( method="quad", variant="B") == approx(0.2617378574, rel=1e-8) assert mvtest.expected_loss_relative_vs_all( method="MLHS", variant="B") == approx(0.2617378574, rel=1e-2) assert mvtest.expected_loss_relative_vs_all( method="MC", variant="B") == approx(0.2617378574, rel=1e-1)
def test_gamma_mv_probability_vs_all(): models = { "A": GammaModel(shape=25, rate=10000), "B": GammaModel(shape=30, rate=10000), "C": GammaModel(shape=40, rate=11000) } mvtest = GammaMVTest(models, 1000000, 42) assert mvtest.probability_vs_all( method="quad", variant="B") == approx(0.1969431392, rel=1e-8) assert mvtest.probability_vs_all( method="MLHS", variant="B") == approx(0.1969431392, rel=1e-2) assert mvtest.probability_vs_all( method="MC", variant="B") == approx(0.1969431392, rel=1e-1)
def test_gamma_ab_expected_loss_relative_ci_large_params(): modelA = GammaModel(shape=2500, rate=1000000) modelB = GammaModel(shape=3000, rate=1000000) abtest = GammaABTest(modelA, modelB, 1000000, 42) assert abtest.expected_loss_relative_ci( method="MC", variant="A") == approx( (0.1477770747, 0.2547302435), rel=1e-2) assert abtest.expected_loss_relative_ci( method="asymptotic", variant="A") == approx( (0.1477770747, 0.2547302435), rel=1e-1) assert abtest.expected_loss_relative_ci( method="MC", variant="B") == approx( (-0.2030159429, -0.1287506763), rel=1e-2) assert abtest.expected_loss_relative_ci( method="asymptotic", variant="B") == approx( (-0.2030159429, -0.1287506763), rel=1e-1)
def test_gamma_ab_expected_loss(): modelA = GammaModel(shape=25, rate=10000) modelB = GammaModel(shape=30, rate=10000) abtest = GammaABTest(modelA, modelB, 1000000, 42) assert abtest.expected_loss( method="exact", variant="A") == approx(0.0006094353823, rel=1e-8) assert abtest.expected_loss( method="MC", variant="A") == approx(0.0006094353823, rel=1e-2) assert abtest.expected_loss( method="exact", variant="B") == approx(0.0001094353823, rel=1e-8) assert abtest.expected_loss( method="MC", variant="B") == approx(0.0001094353823, rel=1e-2) assert abtest.expected_loss(method="exact", variant="all") == approx( [0.0006094353823, 0.0001094353823], rel=1e-8) assert abtest.expected_loss(method="MC", variant="all") == approx( [0.0006094353823, 0.0001094353823], rel=1e-2)
def test_gamma_ab_probability(): modelA = GammaModel(shape=25, rate=10000) modelB = GammaModel(shape=30, rate=10000) abtest = GammaABTest(modelA, modelB, 1000000, 42) assert abtest.probability( method="exact", variant="A") == approx(0.2483087176, rel=1e-8) assert abtest.probability( method="MC", variant="A") == approx(0.2483087176, rel=1e-2) assert abtest.probability( method="exact", variant="B") == approx(0.7516912823, rel=1e-8) assert abtest.probability( method="MC", variant="B") == approx(0.7516912823, rel=1e-2) assert abtest.probability(method="exact", variant="all") == approx( [0.2483087176, 0.7516912823], rel=1e-8) assert abtest.probability(method="MC", variant="all") == approx( [0.2483087176, 0.7516912823], rel=1e-2)
def test_gamma_ab_expected_loss_relative(): modelA = GammaModel(shape=25, rate=10000) modelB = GammaModel(shape=30, rate=10000) abtest = GammaABTest(modelA, modelB, 1000000, 42) assert abtest.expected_loss_relative( method="exact", variant="A") == approx(0.25, rel=1e-8) assert abtest.expected_loss_relative( method="MC", variant="A") == approx(0.25, rel=1e-2) assert abtest.expected_loss_relative( method="exact", variant="B") == approx(-0.1379310344, rel=1e-8) assert abtest.expected_loss_relative( method="MC", variant="B") == approx(-0.1379310344, rel=1e-2) assert abtest.expected_loss_relative( method="exact", variant="all") == approx( [0.25, -0.1379310344], rel=1e-8) assert abtest.expected_loss_relative( method="MC", variant="all") == approx([0.25, -0.1379310344], rel=1e-2)
def test_gamma_ab_expected_loss_ci(): modelA = GammaModel(shape=25, rate=10000) modelB = GammaModel(shape=30, rate=10000) abtest = GammaABTest(modelA, modelB, 1000000, 42) assert abtest.expected_loss_ci(method="MC", variant="A") == approx( [-0.00071135, 0.00171985], rel=1e-2) assert abtest.expected_loss_ci(method="asymptotic", variant="A") == approx( [-0.00071135, 0.00171985], rel=1e-1) assert abtest.expected_loss_ci(method="MC", variant="B") == approx( [-0.00171985, 0.00071135], rel=1e-2) assert abtest.expected_loss_ci(method="asymptotic", variant="B") == approx( [-0.00171985, 0.00071135], rel=1e-1) ci = abtest.expected_loss_ci(method="MC", variant="all") assert ci[0] == approx([-0.00071135, 0.00171985], rel=1e-2) assert ci[1] == approx([-0.00171985, 0.00071135], rel=1e-2) ci = abtest.expected_loss_ci(method="asymptotic", variant="all") assert ci[0] == approx([-0.00071135, 0.00171985], rel=1e-1) assert ci[1] == approx([-0.00171985, 0.00071135], rel=1e-1)
def test_gamma_model_priors(): model = GammaModel(shape=0.1, rate=0.2) assert model.shape_posterior == model.shape assert model.rate_posterior == model.rate
def test_gamma_model_rate_positive(): with raises(ValueError): GammaModel(rate=0.0)
def test_gamma_model_shape_positive(): with raises(ValueError): GammaModel(shape=-0.1)