def test_gamma_uninformed(control_rate, variant1_rate, variant2_rate,
control_variance, variant1_variance,
variant2_variance, control_name, variant1_name,
variant2_name, log):
"""Create data and run a two variant report for a continuous metric.
While testing the uninformed prior for a gamma distribution.
"""
rawdata = dh.create_continuous_data(
[control_rate, variant1_rate, variant2_rate],
[control_variance, variant1_variance, variant2_variance],
[control_name, variant1_name, variant2_name],
metric_name='total_premium',
log=log)
prior = 'log-normal' if log else 'normal'
premium = ab(rawdata,
metric='total_premium',
prior_info='uninformed',
prior_function=prior,
debug=True,
compare_variants=True,
control_bucket_name=control_name,
samples=1000)
premium.fit()
premium.plot()
def test_one_variant_continuous(control_rate, variant_rate, control_variance,
variant_variance, control_name, variant_name):
"""Create data and run a one variant report for a continuous metric."""
raw_data_premium = dh.create_continuous_data(
[control_rate, variant_rate], [control_variance, variant_variance],
[control_name, variant_name],
metric_name='total_premium')
premium_test = ab(raw_data_premium,
metric='total_premium',
control_bucket_name=control_name,
prior_info='informed',
prior_function='log-normal',
debug=True,
samples=1000)
premium_test.fit()
premium_test.plot()
def test_three_variants_continuous():
"""Create data and run a three variant report for a continuous metric."""
rawdata = dh.create_continuous_data(
[600, 610, 615, 620], [1.5, 1.5, 1.5, 1.5],
['control', 'variant_1', 'variant_2', 'variant_3'],
metric_name='total_premium')
premium = ab(rawdata,
metric='total_premium',
prior_info='informed',
prior_function='log-normal',
debug=True,
control_bucket_name='control',
compare_variants=True,
samples=1000)
premium.fit()
premium.plot()
def test_specified_prior_continuous():
"""E2E test a user specified prior for continuous."""
prior = {'mean': 650, 'var': 1.5}
rawdata = dh.create_continuous_data([600, 610, 615], [1.5, 1.5, 1.5],
['control', 'variant_1', 'variant_2'],
metric_name='total_premium')
premium = ab(rawdata,
metric='total_premium',
prior_info='specified',
prior_function='log-normal',
debug=True,
control_bucket_name='control',
compare_variants=True,
prior_parameters=prior,
samples=1000)
premium.fit()
premium.plot()
def test_four_variants_continuous():
"""Create data and run a four variant report for a continuous metric."""
rawdata = dh.create_continuous_data(
[600, 610, 615, 620, 625], [1.1, 1.1, 1.1, 1.1, 1.1],
['control', 'variant_1', 'variant_2', 'variant_3', 'variant_4'],
metric_name='total_premium',
sample_length=1000)
premium = ab(rawdata,
metric='total_premium',
prior_info='informed',
prior_function='log-normal',
debug=True,
control_bucket_name='control',
compare_variants=False,
samples=1000)
premium.fit()
premium.plot(lift_plot_flag=True)
def test_specified_prior_normal():
"""E2E test a user specified prior for continuous, log=False."""
prior = {'mean': 650, 'var': 30000}
rawdata = dh.create_continuous_data([600, 601, 602], [30, 30, 30],
['control', 'variant_1', 'variant_2'],
metric_name='total_premium',
log=False)
premium = ab(rawdata,
metric='total_premium',
prior_info='specified',
prior_function='normal',
debug=True,
control_bucket_name='control',
compare_variants=True,
prior_parameters=prior,
samples=1000)
premium.fit()
premium.plot(lift_plot_flag=True)