def test_hypothesis_test_suite():
exp = Experiment(proportions_data_large, name='test')
# run 'A/A' test (should never reject null)
test_aa = HypothesisTest(
metric='metric',
control='A',
variation='A',
hypothesis='larger',
inference_method='proportions_delta',
)
# run A/B test
test_ab = HypothesisTest(metric='metric',
control='A',
variation='B',
hypothesis='larger',
inference_method='proportions_delta')
test_suite = HypothesisTestSuite([test_aa, test_ab], correction_method='b')
test_suite_results = exp.run_test_suite(test_suite)
assert test_suite_results.corrected_results[
0].correction_method == 'bonferroni'
assert test_suite_results.original_results[0].correction_method is None
# corrected alpha should be smaller
alpha_orig = test_suite_results.original_results[0].alpha
alpha_corrected = test_suite_results.corrected_results[0].alpha
assert alpha_orig > alpha_corrected
assert not test_suite_results.corrected_results[0].accept_hypothesis
assert test_suite_results.corrected_results[1].accept_hypothesis
def test_rates_ratio_aa(counts_data):
exp = Experiment(data=counts_data)
aa_test = HypothesisTest(inference_method='rates_ratio',
metric='metric',
control='A',
variation='A')
aa_results = exp.run_test(aa_test)
assert not aa_results.accept_hypothesis
def test_rates_ratio_larger(counts_data):
exp = Experiment(data=counts_data)
ab_test = HypothesisTest(inference_method='rates_ratio',
metric='metric',
hypothesis='larger',
control='A',
variation='C')
ab_results = exp.run_test(ab_test)
assert ab_results.accept_hypothesis
def test_bayesian_binomial(proportions_data_large):
exp = Experiment(data=proportions_data_large)
test = HypothesisTest(inference_method='binomial',
metric='metric',
control='A',
variation='C')
inference_kwargs = dict(inference_method='sample')
test_results = exp.run_test(test, inference_kwargs=inference_kwargs)
assert pytest.approx(test_results.prob_greater, rel=.1, abs=.01) == 1.
def test_bayesian_gaussian(means_data):
exp = Experiment(data=means_data)
test = HypothesisTest(inference_method='gaussian',
metric='metric',
control='A',
variation='F')
inference_kwargs = dict(inference_method='sample')
test_results = exp.run_test(test, inference_kwargs=inference_kwargs)
assert pytest.approx(test_results.prob_greater, rel=.1, abs=.01) == 1.
def test_frequentist_methods():
exp = Experiment(data=proportions_data_large)
test = HypothesisTest(metric='metric',
control='A',
variation='C',
inference_method='proportions_delta')
test_results = exp.run_test(test)
assert test_results.accept_hypothesis
def test_empty_observations_exception(proportions_data_large):
exp = Experiment(data=proportions_data_large)
# no F variation in proportions_data_large
with pytest.raises(ValueError):
test = HypothesisTest(inference_method='binomial',
metric='metric',
control='A',
variation='F')
inference_kwargs = dict(inference_method='sample')
_ = exp.run_test(test, inference_kwargs=inference_kwargs)
def test_proportions_delta_aa(proportions_data_small):
exp = Experiment(proportions_data_small, name='proportions-test')
# run A/A test
test_aa = HypothesisTest(metric='metric',
control='A',
variation='A',
hypothesis='larger',
inference_method='proportions_delta')
results_aa = exp.run_test(test_aa)
assert not results_aa.accept_hypothesis
def test_means_delta_experiment_aa(means_data):
exp = Experiment(means_data, name='means-test')
test_ab = HypothesisTest(metric='metric',
control='A',
variation='A',
hypothesis='larger',
inference_method='means_delta')
results_ab = exp.run_test(test_ab)
assert results_ab.test_statistic == 'z'
assert not results_ab.accept_hypothesis
def test_means_delta_experiment_t(means_data):
"""Small sample sizes defautl to t-tests"""
exp = Experiment(means_data.sample(29), name='means-test')
test_ab = HypothesisTest(metric='metric',
control='A',
variation='A',
hypothesis='unequal',
inference_method='means_delta')
results_ab = exp.run_test(test_ab)
assert results_ab.test_statistic == 't'
def test_small_bootstrap_larger_ab_test(proportions_data_small):
exp = Experiment(proportions_data_small, name='proportions-test')
# run A/B test
test_ab = HypothesisTest(metric='metric',
control='A',
variation='D',
hypothesis='smaller',
inference_method='bootstrap')
results_ab = exp.run_test(test_ab)
assert not results_ab.accept_hypothesis
def test_proportions_delta_ab_unequal(proportions_data_small):
exp = Experiment(proportions_data_small, name='proportions-test')
# run A/B test
test_ab = HypothesisTest(metric='metric',
control='A',
variation='F',
hypothesis='unequal',
inference_method='proportions_delta')
results_ab = exp.run_test(test_ab)
assert results_ab.test_statistic == 'z'
assert results_ab.accept_hypothesis
def test_small_default_bootstrap_unequal_ab_test(proportions_data_large):
exp = Experiment(proportions_data_large, name='proportions-test')
# run A/B test
test_ab = HypothesisTest(metric='metric',
control='A',
variation='B',
hypothesis='unequal',
inference_method='bootstrap')
results_ab = exp.run_test(test_ab)
assert results_ab.test_statistic == 'mean'
assert results_ab.accept_hypothesis
def test_small_median_bootstrap_aa_test(proportions_data_small):
exp = Experiment(proportions_data_small, name='proportions-test')
# run A/B test
test_ab = HypothesisTest(
metric='metric',
control='A',
variation='A',
hypothesis='unequal',
inference_method='bootstrap',
statistic_function=median,
)
results_ab = exp.run_test(test_ab)
assert results_ab.test_statistic == 'median'
assert not results_ab.accept_hypothesis
def test_custom_metric():
exp = Experiment(means_data, name='means-test')
def custom_metric(row):
return 4 + np.random.rand(
) if row['treatment'] != 'A' else np.random.rand()
test_ab = HypothesisTest(metric=CustomMetric(custom_metric),
control='A',
variation='F',
hypothesis='larger',
inference_method='means_delta')
results_ab = exp.run_test(test_ab)
results_ab.to_dataframe()
assert results_ab.test_statistic == 'z'
assert results_ab.accept_hypothesis
def test_large_proportions_delta_expermiment(proportions_data_large):
exp = Experiment(proportions_data_large, name='proportions-test')
# run 'A/A' test
test_aa = HypothesisTest(metric='metric',
control='A',
variation='A',
hypothesis='larger',
inference_method='proportions_delta')
results_aa = exp.run_test(test_aa)
assert results_aa.test_statistic == 'z'
assert not results_aa.accept_hypothesis
# run A/B test
test_ab = HypothesisTest(metric='metric',
control='A',
variation='B',
hypothesis='larger',
inference_method='proportions_delta')
results_ab = exp.run_test(test_ab)
assert results_ab.test_statistic == 'z'
assert results_ab.accept_hypothesis
