def test_bayes_precision_delta(self):
ndecimals = 5
res = self.getExperiment(['normal_same'
]).delta(method='bayes_precision',
num_iters=2000)
variants = find_list_of_dicts_element(res['kpis'], 'name',
'normal_same', 'variants')
aStats = find_list_of_dicts_element(variants, 'name', 'A',
'delta_statistics')
self.assertNumericalEqual(aStats['delta'], 0.033053, ndecimals)
self.assertEqual(aStats['stop'], True, ndecimals)
self.assertEqual(aStats['number_of_iterations'], 2000, ndecimals)
self.assertNumericalEqual(aStats['confidence_interval'][0]['value'],
-0.00829, ndecimals)
self.assertNumericalEqual(aStats['confidence_interval'][1]['value'],
0.07127, ndecimals)
self.assertEqual(aStats['treatment_sample_size'], 6108)
self.assertEqual(aStats['control_sample_size'], 3892)
self.assertNumericalEqual(aStats['treatment_mean'], 0.025219,
ndecimals)
self.assertNumericalEqual(aStats['control_mean'], -0.007833, ndecimals)
self.assertNumericalEqual(aStats['statistical_power'], 0.36401,
ndecimals)
def test__delta__computation_assumed_normal(self):
"""
Result of delta() assuming normality equals expected result.
"""
# Computing delta assumed normal
res = statx.delta(self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2],
percentiles=[2.5, 97.5],
assume_normal=True)
# Checking if mean has right value
self.assertAlmostEqual(res['delta'], -0.28923076923075541)
value025 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 2.5, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 97.5, 'value')
# Checking if lower percentile has right value
self.assertAlmostEqual(value025, -0.53770569567692295)
# Checking if uper percentile has right value
self.assertAlmostEqual(value975, -0.040755842784587965)
# Checking if sample size 1 is correct
self.assertEqual(res['treatment_sample_size'], 65)
# Checking if sample size 2 is correct
self.assertEqual(res['control_sample_size'], 65)
def test_group_sequential_delta(self):
ndecimals = 5
res = self.getExperiment(['normal_same'
]).delta(method='group_sequential')
variants = find_list_of_dicts_element(res['kpis'], 'name',
'normal_same', 'variants')
aStats = find_list_of_dicts_element(variants, 'name', 'A',
'delta_statistics')
self.assertNumericalEqual(aStats['delta'], 0.033053, ndecimals)
self.assertNumericalEqual(aStats['confidence_interval'][0]['value'],
-0.007135, ndecimals)
self.assertNumericalEqual(aStats['confidence_interval'][1]['value'],
0.073240, ndecimals)
self.assertEqual(aStats['treatment_sample_size'], 6108)
self.assertEqual(aStats['control_sample_size'], 3892)
self.assertNumericalEqual(aStats['treatment_mean'], 0.025219,
ndecimals)
self.assertNumericalEqual(aStats['control_mean'], -0.007833, ndecimals)
self.assertNumericalEqual(aStats['statistical_power'], 0.36401,
ndecimals)
def test_group_sequential_multi_test_correction(self):
"""
Test group sequential with multiple correction
"""
res = es.group_sequential(self.rand_s1,
self.rand_s2,
multi_test_correction=True,
num_tests=25)
self.assertEqual(res['stop'], True)
self.assertAlmostEqual(res['delta'], -0.15887364780635896)
value025 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 0.1, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 99.9, 'value')
np.testing.assert_almost_equal(value025,
-0.29416175390519078,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.023585541707525692,
decimal=5)
self.assertEqual(res['treatment_sample_size'], 1000)
self.assertEqual(res['control_sample_size'], 1000)
self.assertAlmostEqual(res['treatment_mean'], -0.045256707490195384)
self.assertAlmostEqual(res['control_mean'], 0.11361694031616358)
def test_bayes_factor_poisson(self):
"""
Check the Bayes factor function for Poisson distributions.
"""
res= es.bayes_factor(self.rand_s3, self.rand_s4, distribution='poisson', num_iters=2000)
self.assertEqual (res['stop'], True)
self.assertAlmostEqual (res['delta'], -1.9589999999999999)
value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value')
self.assertAlmostEqual (value025, -2.0713281392132465)
self.assertAlmostEqual (value975, -1.8279692168150592)
self.assertEqual (res['treatment_sample_size'], 1000)
self.assertEqual (res['control_sample_size'], 1000)
self.assertAlmostEqual (res['treatment_mean'], 0.96599999999999997)
self.assertAlmostEqual (res['control_mean'], 2.9249999999999998)
def test_bayes_factor(self):
"""
Check the Bayes factor function.
"""
res = es.bayes_factor(self.rand_s1, self.rand_s2, num_iters=2000)
self.assertEqual (res['stop'], True)
self.assertAlmostEqual (res['delta'], -0.15887364780635896)
value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value')
self.assertAlmostEqual (value025, -0.24293384641452503)
self.assertAlmostEqual (value975, -0.075064346336461404)
self.assertEqual (res['treatment_sample_size'], 1000)
self.assertEqual (res['control_sample_size'], 1000)
self.assertAlmostEqual (res['treatment_mean'], -0.045256707490195384)
self.assertAlmostEqual (res['control_mean'], 0.11361694031616358)
def test_group_sequential(self):
"""
Check the group sequential function.
"""
res = es.group_sequential(self.rand_s1, self.rand_s2)
self.assertEqual (res['stop'], True)
self.assertAlmostEqual (res['delta'], -0.15887364780635896)
value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value')
np.testing.assert_almost_equal (value025, -0.24461812530841959, decimal=5)
np.testing.assert_almost_equal (value975, -0.07312917030429833, decimal=5)
self.assertEqual (res['treatment_sample_size'], 1000)
self.assertEqual (res['control_sample_size'], 1000)
self.assertAlmostEqual (res['treatment_mean'], -0.045256707490195384)
self.assertAlmostEqual (res['control_mean'], 0.11361694031616358)
def test__delta__2percentiles_no_tests(self):
"""
Percentiles of delta() for sga are corrected for no tests (1 as a default)
"""
res = statx.delta(self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2],
percentiles=[2.5, 97.5],
assume_normal=True,
num_tests=1)
value025 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 2.5, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 97.5, 'value')
self.assertAlmostEqual(value025, -0.53770569567692295)
self.assertAlmostEqual(value975, -0.040755842784587965)
def test__delta__2percentiles_25_tests(self):
"""
Percentiles of delta() for sga are corrected for 25 tests
"""
res = statx.delta(self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2],
percentiles=[2.5, 97.5],
assume_normal=True,
num_tests=25)
value025 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 0.1, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 99.9, 'value')
self.assertAlmostEqual(value025, -0.68544085117601006)
self.assertAlmostEqual(value975, 0.1069793127145035)
def test_sga_date(self):
exp = self.getExperiment([self.derived_kpi_1['name']],
[self.derived_kpi_1])
sga_result = exp.sga_date()
self.assertEqual(len(sga_result), 417)
numerical_dimension_name = find_list_of_dicts_element(
sga_result, "segment", "['2016-01-21']", "dimension")
self.assertEqual(numerical_dimension_name, 'date')
def test_group_sequential_actual_size_larger_than_estimated(self):
"""
Check the group sequential function with wrong input,
such that the actual data size is already larger than estimated sample size.
"""
res = es.group_sequential(self.rand_s1,
self.rand_s2,
estimated_sample_size=100)
value025 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 2.5, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 97.5, 'value')
np.testing.assert_almost_equal(value025,
-0.24461812530841959,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.07312917030429833,
decimal=5)
def test_sga_multi_dim_multi_bins(self):
exp = self.getExperiment([self.derived_kpi_1['name']],
[self.derived_kpi_1])
dimension_to_bin = {
"normal_same": [
Bin("numerical", 1, 2, True, False),
Bin("numerical", 2, 3, True, False)
],
"feature":
[Bin("categorical", ["has"]),
Bin("categorical", ["non"])]
}
sga_result = exp.sga(dimension_to_bin)
self.assertEqual(len(sga_result), 4)
numerical_dimension_name = find_list_of_dicts_element(
sga_result, "segment", "[1, 2)", "dimension")
self.assertEqual(numerical_dimension_name, 'normal_same')
categorical_dimension_name = find_list_of_dicts_element(
sga_result, "segment", "['non']", "dimension")
self.assertEqual(categorical_dimension_name, 'feature')
def test_bayes_precision_delta(self):
ndecimals = 5
res = self.getExperiment(['normal_same']).delta(method='bayes_precision', num_iters=2000)
variants = find_list_of_dicts_element(res['kpis'], 'name', 'normal_same', 'variants')
aStats = find_list_of_dicts_element(variants, 'name', 'A', 'delta_statistics')
self.assertNumericalEqual(aStats['delta'], 0.033053, ndecimals)
self.assertEqual(aStats['stop'], True, ndecimals)
self.assertEqual(aStats['number_of_iterations'], 2000, ndecimals)
#
# this can result in different numerical values depending on Python version
#
# self.assertNumericalEqual(aStats['confidence_interval'][0]['value'], -0.007079081, ndecimals)
# self.assertNumericalEqual(aStats['confidence_interval'][1]['value'], 0.072703576, ndecimals)
self.assertEqual(aStats['treatment_sample_size'], 6108)
self.assertEqual(aStats['control_sample_size'], 3892)
self.assertNumericalEqual(aStats['treatment_mean'], 0.025219, ndecimals)
self.assertNumericalEqual(aStats['control_mean'], -0.007833, ndecimals)
def test_find_dict_element(self):
list_of_dicts = [{
'bla': 1,
'blu': 2
}, {
'bla': 3,
'blu': 4
}, {
'bla': 5,
'blu': 6
}]
self.assertEqual(
util.find_list_of_dicts_element(list_of_dicts, 'bla', 5, 'blu'), 6)
