def test_gamma_uninformed(control_rate, variant1_rate, variant2_rate,
control_variance, variant1_variance,
variant2_variance, control_name, variant1_name,
variant2_name, log):
"""Create data and run a two variant report for a continuous metric.
While testing the uninformed prior for a gamma distribution.
"""
rawdata = dh.create_continuous_data(
[control_rate, variant1_rate, variant2_rate],
[control_variance, variant1_variance, variant2_variance],
[control_name, variant1_name, variant2_name],
metric_name='total_premium',
log=log)
prior = 'log-normal' if log else 'normal'
premium = ab(rawdata,
metric='total_premium',
prior_info='uninformed',
prior_function=prior,
debug=True,
compare_variants=True,
control_bucket_name=control_name,
samples=1000)
premium.fit()
premium.plot()
def test_single_plot_ecdf_unit():
"""Test plot_ecdf.
Use a poisson metric to test the public version of plot_ecdf,
with various subsets of the variants.
"""
rawdata = dh.create_poisson_data([15, 17, 20],
['control', 'variant_1', 'variant_2'],
metric_name='visits',
sample_length=3000)
visits = ab(rawdata,
metric='visits',
prior_info='informed',
prior_function='poisson',
debug=True,
control_bucket_name='control',
compare_variants=True,
samples=3000)
visits.fit()
visits.plot_ecdf('control', 'variant_1')
visits.plot_ecdf('variant_1', 'variant_2')
visits.plot_ecdf('control', 'variant_2')
visits.plot_ecdf('variant_1', 'control')
visits.plot_ecdf('variant_2', 'variant_1')
visits.plot_ecdf('variant_2', 'control')
print('PASSED: test_single_plot_ecdf_unit')
def test_one_variant_conversion(control_rate, variant_rate, control_name,
variant_name):
"""Create data and run a one variant report for a conversion metric."""
raw_data_auto_bind = dh.create_conversion_data(
[control_rate, variant_rate], [control_name, variant_name],
metric_name='bind')
auto_bind = ab(raw_data_auto_bind,
metric='bind',
control_bucket_name=control_name,
prior_info='uninformed',
prior_function='beta',
debug=True,
samples=1000)
auto_bind.fit()
auto_bind.plot()
def test_one_variant_continuous(control_rate, variant_rate, control_variance,
variant_variance, control_name, variant_name):
"""Create data and run a one variant report for a continuous metric."""
raw_data_premium = dh.create_continuous_data(
[control_rate, variant_rate], [control_variance, variant_variance],
[control_name, variant_name],
metric_name='total_premium')
premium_test = ab(raw_data_premium,
metric='total_premium',
control_bucket_name=control_name,
prior_info='informed',
prior_function='log-normal',
debug=True,
samples=1000)
premium_test.fit()
premium_test.plot()
def test_three_variants_continuous():
"""Create data and run a three variant report for a continuous metric."""
rawdata = dh.create_continuous_data(
[600, 610, 615, 620], [1.5, 1.5, 1.5, 1.5],
['control', 'variant_1', 'variant_2', 'variant_3'],
metric_name='total_premium')
premium = ab(rawdata,
metric='total_premium',
prior_info='informed',
prior_function='log-normal',
debug=True,
control_bucket_name='control',
compare_variants=True,
samples=1000)
premium.fit()
premium.plot()
def test_specified_prior_continuous():
"""E2E test a user specified prior for continuous."""
prior = {'mean': 650, 'var': 1.5}
rawdata = dh.create_continuous_data([600, 610, 615], [1.5, 1.5, 1.5],
['control', 'variant_1', 'variant_2'],
metric_name='total_premium')
premium = ab(rawdata,
metric='total_premium',
prior_info='specified',
prior_function='log-normal',
debug=True,
control_bucket_name='control',
compare_variants=True,
prior_parameters=prior,
samples=1000)
premium.fit()
premium.plot()
def test_specified_prior_conversion():
"""E2E test a user specified prior for conversion."""
raw_data_2vars = dh.create_conversion_data(
[.22, .23, .235], ['control', 'variant_1', 'variant_2'],
metric_name='conversion')
prior = {'alpha': 22, 'beta': 100 - 22}
site_conversion = ab(raw_data_2vars,
metric='conversion',
prior_info='specified',
prior_function='beta',
debug=True,
control_bucket_name='control',
compare_variants=True,
prior_parameters=prior,
samples=1000)
site_conversion.fit()
site_conversion.plot(lift_plot_flag=True)
def test_four_variants_continuous():
"""Create data and run a four variant report for a continuous metric."""
rawdata = dh.create_continuous_data(
[600, 610, 615, 620, 625], [1.1, 1.1, 1.1, 1.1, 1.1],
['control', 'variant_1', 'variant_2', 'variant_3', 'variant_4'],
metric_name='total_premium',
sample_length=1000)
premium = ab(rawdata,
metric='total_premium',
prior_info='informed',
prior_function='log-normal',
debug=True,
control_bucket_name='control',
compare_variants=False,
samples=1000)
premium.fit()
premium.plot(lift_plot_flag=True)
def test_specified_prior_normal():
"""E2E test a user specified prior for continuous, log=False."""
prior = {'mean': 650, 'var': 30000}
rawdata = dh.create_continuous_data([600, 601, 602], [30, 30, 30],
['control', 'variant_1', 'variant_2'],
metric_name='total_premium',
log=False)
premium = ab(rawdata,
metric='total_premium',
prior_info='specified',
prior_function='normal',
debug=True,
control_bucket_name='control',
compare_variants=True,
prior_parameters=prior,
samples=1000)
premium.fit()
premium.plot(lift_plot_flag=True)
def test_specified_prior_poisson_alpha_beta():
"""E2E test a user specified prior for a beta distribution."""
prior = {'alpha': 8, 'beta': 2}
rawdata = dh.create_poisson_data([15, 17, 20],
['control', 'variant_1', 'variant_2'],
metric_name='visits',
sample_length=3000)
visits = ab(rawdata,
metric='visits',
prior_info='specified',
prior_function='poisson',
debug=True,
control_bucket_name='control',
compare_variants=True,
prior_parameters=prior,
samples=3000)
visits.fit()
visits.plot(lift_plot_flag=True)
def test_two_variants_poisson(control_rate, variant1_rate, variant2_rate,
control_name, variant1_name, variant2_name):
"""Create data and run a two variant report for a poisson metric."""
rawdata = dh.create_poisson_data(
[control_rate, variant1_rate, variant2_rate],
[control_name, variant1_name, variant2_name],
metric_name='visits',
sample_length=3000)
rawdata = rawdata.sample(frac=1).reset_index(drop=True)
visits = ab(rawdata,
metric='visits',
prior_info='informed',
prior_function='poisson',
debug=True,
control_bucket_name=control_name,
compare_variants=True,
samples=3000)
visits.fit()
visits.plot()