def test_bernoulli_model_stats():
model = BernoulliModel(alpha=4, beta=6)
assert model.pppdf(0) == approx(0.6)
assert model.pppdf(1) == approx(0.4)
assert model.ppmean() == approx(0.4)
assert model.ppvar() == approx(0.24)
def test_bernoulli_mv_check_control():
models = {
"B": BernoulliModel(name="variant 1", alpha=1, beta=1),
"C": BernoulliModel(name="variant 2", alpha=1, beta=1)
}
with raises(ValueError):
BernoulliMVTest(models=models)
def test_bernoulli_model_update():
model = BernoulliModel(alpha=1, beta=1)
data = np.array([0, 0, 0, 1, 1])
model.update(data)
assert model.n_samples_ == 5
assert model.n_success_ == 2
def test_bernoulli_mv_check_update():
models = {
"A": BernoulliModel(name="control", alpha=1, beta=1),
"B": BernoulliModel(name="variant 2", alpha=1, beta=1)
}
mvtest = BernoulliMVTest(models=models)
with raises(ValueError):
data = np.array([0, 0, 0, 1, 1])
mvtest.update(data=data, variant="C")
def test_experiment_bernoulli_expected_loss():
modelA = BernoulliModel(name="control", alpha=1, beta=1)
modelB = BernoulliModel(name="variation", alpha=1, beta=1)
test = BernoulliMVTest({"A": modelA, "B": modelB})
experiment = Experiment(name="CTR", test=test,
stopping_rule="expected_loss", epsilon=1e-5)
data_A = stats.bernoulli(p=0.0223).rvs(size=1500, random_state=42)
data_B = stats.bernoulli(p=0.1128).rvs(size=1100, random_state=42)
experiment.run_update(**{"A": data_A, "B": data_B})
assert experiment.termination
assert experiment.winner == "B"
def test_experiment_summary():
modelA = BernoulliModel(name="control", alpha=1, beta=1)
modelB = BernoulliModel(name="variation", alpha=1, beta=1)
test = BernoulliMVTest({"A": modelA, "B": modelB})
experiment = Experiment(name="CTR", test=test, stopping_rule="probability",
epsilon=0.99)
data_A = stats.bernoulli(p=0.0223).rvs(size=1500, random_state=42)
data_B = stats.bernoulli(p=0.1128).rvs(size=1100, random_state=42)
experiment.run_update(**{"A": data_A, "B": data_B})
with raises(TypeError):
experiment_summary(None)
experiment.summary()
def test_experiment_bernoulli_probability_vs_all():
modelA = BernoulliModel(name="control", alpha=1, beta=1)
modelB = BernoulliModel(name="variation", alpha=1, beta=1)
modelC = BernoulliModel(name="variation", alpha=1, beta=1)
test = BernoulliMVTest({"A": modelA, "B": modelB, "C": modelC})
experiment = Experiment(name="CTR", test=test,
stopping_rule="probability_vs_all", epsilon=0.99)
data_A = stats.bernoulli(p=0.0223).rvs(size=1500, random_state=42)
data_B = stats.bernoulli(p=0.1128).rvs(size=1100, random_state=42)
data_C = stats.bernoulli(p=0.0528).rvs(size=900, random_state=42)
experiment.run_update(**{"A": data_A, "B": data_B, "C": data_C})
assert experiment.termination
assert experiment.winner == "B"
def test_experiment_bernoulli_one_update():
modelA = BernoulliModel(name="control", alpha=1, beta=1)
modelB = BernoulliModel(name="variation", alpha=1, beta=1)
test = BernoulliMVTest({"A": modelA, "B": modelB})
experiment = Experiment(name="CTR", test=test, stopping_rule="probability",
epsilon=0.99)
with experiment as e:
while not e.termination:
data_A = stats.bernoulli(p=0.0223).rvs()
data_B = stats.bernoulli(p=0.1128).rvs()
e.run_update(**{"A": data_A, "B": data_B})
assert experiment.termination
assert experiment.winner == "B"
def test_experiment_bernoulli_probability_min_samples():
modelA = BernoulliModel(name="control", alpha=1, beta=1)
modelB = BernoulliModel(name="variation", alpha=1, beta=1)
abtest = BernoulliMVTest({"A": modelA, "B": modelB})
experiment = Experiment(name="CTR", test=abtest,
stopping_rule="probability", epsilon=0.99,
min_n_samples=1000)
with experiment as e:
data_A = stats.bernoulli(p=0.0223).rvs(size=90, random_state=42)
data_B = stats.bernoulli(p=0.1128).rvs(size=50, random_state=42)
e.run_update(**{"A": data_A, "B": data_B})
assert e.termination is False
def test_experiment_load():
modelA = BernoulliModel(name="control", alpha=1, beta=1)
modelB = BernoulliModel(name="variation", alpha=1, beta=1)
abtest = BernoulliMVTest({"A": modelA, "B": modelB})
experiment = Experiment(name="CTR", test=abtest,
stopping_rule="probability", epsilon=0.99,
min_n_samples=100)
with raises(TypeError):
experiment.load(None)
experiment.load("experiment_ctr.pkl")
assert experiment.termination is True
assert experiment.winner == "B"
assert experiment.status == "winner B"
def test_experiment_input():
modelA = BernoulliModel(name="control", alpha=1, beta=1)
modelB = BernoulliModel(name="variation", alpha=1, beta=1)
test = BernoulliMVTest({"A": modelA, "B": modelB})
with raises(ValueError):
Experiment(name="CTR", test=test, stopping_rule="new_stopping_rule")
with raises(ValueError):
Experiment(name="CTR", test=test, min_n_samples=-10)
with raises(ValueError):
Experiment(name="CTR", test=test, max_n_samples=-10)
with raises(ValueError):
Experiment(name="CTR", test=test, min_n_samples=100, max_n_samples=10)
with raises(TypeError):
Experiment(name="CTR", test=None)
def test_geometric_ab_check_models():
modelA = BernoulliModel(alpha=1, beta=1)
modelB = GeometricModel(alpha=1, beta=1)
with raises(TypeError):
GeometricABTest(modelA=modelA, modelB=modelB)
"""
Example Bayesian model with Bernoulli distribution.
"""
# Guillermo Navas-Palencia <*****@*****.**>
# Copyright (C) 2019
import scipy.stats as st
from cprior.models import BernoulliModel
from cprior.models import BernoulliABTest
# Two model variants A and B, and build A/B test
modelA = BernoulliModel(alpha=1, beta=1)
modelB = BernoulliModel(alpha=1, beta=1)
test = BernoulliABTest(modelA=modelA, modelB=modelB, simulations=1000000)
# Generate new data and update models
data_A = st.bernoulli(p=0.10).rvs(size=1500, random_state=42)
data_B = st.bernoulli(p=0.11).rvs(size=1600, random_state=42)
test.update_A(data_A)
test.update_B(data_B)
# Compute P[A > B] and P[B > A]
print("P[A > B] = {:.4f}".format(test.probability(variant="A")))
print("P[B > A] = {:.4f}".format(test.probability(variant="B")))
# Compute posterior expected loss given a variant
print("E[max(B - A, 0)] = {:.4f}".format(test.expected_loss(variant="A")))
def test_bernoulli_mv_check_model_input():
modelA = BernoulliModel(alpha=1, beta=1)
modelB = BernoulliModel(alpha=1, beta=1)
with raises(TypeError):
BernoulliMVTest(models=[modelA, modelB])
def test_bernoulli_model_pppdf_x():
model = BernoulliModel(alpha=4, beta=6)
assert model.pppdf([0, 1, 2]) == approx([0.6, 0.4, 0])