def test_bayes_factor_poisson(self):
""" Check the Bayes factor function for Poisson distributions."""
res = es.bayes_factor(self.rand_s3,
self.rand_s4,
distribution='poisson',
num_iters=2000)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean,
0.96599999999999997)
self.assertAlmostEqual(res.control_statistics.mean, 2.9249999999999998)
self.assertAlmostEqual(res.treatment_statistics.variance, 0.868844)
self.assertAlmostEqual(res.control_statistics.variance, 2.901375)
self.assertAlmostEqual(res.delta, -1.9589999999999999)
value025 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 97.5,
'value')
np.testing.assert_almost_equal(value025,
-2.0713281392132465,
decimal=5)
np.testing.assert_almost_equal(value975,
-1.8279692168150592,
decimal=5)
self.assertEqual(res.p, None)
self.assertEqual(res.statistical_power, None)
self.assertEqual(res.stop, True)
def test_one_test_in_suite(self):
ndecimals = 5
res = self.getExperiment().analyze_statistical_test_suite(self.suite_with_one_test)
self.assertEqual(res.correction_method, CorrectionMethod.NONE)
self.assertEqual(len(res.results), 1)
simple_stat_res = res.results[0].result
self.assertAlmostEqual(simple_stat_res.original_test_statistics.delta, 0.033053, ndecimals)
lower_bound_ci = find_value_by_key_with_condition(simple_stat_res.original_test_statistics.confidence_interval,
'percentile', 2.5, 'value')
upper_bound_ci = find_value_by_key_with_condition(simple_stat_res.original_test_statistics.confidence_interval,
'percentile', 97.5, 'value')
self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals)
self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals)
self.assertEqual(simple_stat_res.original_test_statistics.treatment_statistics.sample_size, 6108)
self.assertEqual(simple_stat_res.original_test_statistics.control_statistics.sample_size, 3892)
self.assertAlmostEqual(simple_stat_res.original_test_statistics.treatment_statistics.mean, 0.025219, ndecimals)
self.assertAlmostEqual(simple_stat_res.original_test_statistics.control_statistics.mean, -0.007833, ndecimals)
self.assertAlmostEqual(simple_stat_res.original_test_statistics.statistical_power, 0.36401, ndecimals)
# corrected test statistics should be the same as original test statistics (checks some values)
self.assertAlmostEqual(simple_stat_res.corrected_test_statistics.delta, 0.033053, ndecimals)
self.assertAlmostEqual(simple_stat_res.corrected_test_statistics.statistical_power, 0.36401, ndecimals)
self.assertEqual(simple_stat_res.corrected_test_statistics.treatment_statistics.sample_size, 6108)
self.assertEqual(simple_stat_res.corrected_test_statistics.control_statistics.sample_size, 3892)
def test_bayes_precision(self):
""" Check the bayes_precision function."""
res = es.bayes_precision(self.rand_s1, self.rand_s2, num_iters=2000)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean,
-0.045256707490195384)
self.assertAlmostEqual(res.control_statistics.mean,
0.11361694031616358)
self.assertAlmostEqual(res.treatment_statistics.variance,
0.9742344563121542)
self.assertAlmostEqual(res.control_statistics.variance,
0.9373337542827797)
self.assertAlmostEqual(res.delta, -0.15887364780635896)
value025 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 97.5,
'value')
np.testing.assert_almost_equal(value025,
-0.24293384641452503,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.07506434633646140,
decimal=5)
self.assertEqual(res.p, None)
self.assertEqual(res.statistical_power, None)
self.assertEqual(res.stop, False)
def test_group_sequential(self):
""" Check the group sequential function."""
res = es.group_sequential(self.rand_s1, self.rand_s2)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean,
-0.045256707490195384)
self.assertAlmostEqual(res.control_statistics.mean,
0.11361694031616358)
self.assertAlmostEqual(res.treatment_statistics.variance,
0.9742344563121542)
self.assertAlmostEqual(res.control_statistics.variance,
0.9373337542827797)
self.assertAlmostEqual(res.delta, -0.15887364780635896)
value025 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 97.5,
'value')
np.testing.assert_almost_equal(value025,
-0.24461812530841959,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.07312917030429833,
decimal=5)
self.assertAlmostEqual(res.p, 0.0002863669955157941)
self.assertAlmostEqual(res.statistical_power, 0.9529152504960496)
self.assertEqual(res.stop, True)
def test_group_sequential(self):
ndecimals = 5
res = self.getExperiment().analyze_statistical_test(
self.test_normal_same, test_method='group_sequential')
self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals)
lower_bound_ci = find_value_by_key_with_condition(
res.result.confidence_interval, 'percentile', 2.5, 'value')
upper_bound_ci = find_value_by_key_with_condition(
res.result.confidence_interval, 'percentile', 97.5, 'value')
self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals)
self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals)
self.assertEqual(res.result.treatment_statistics.sample_size, 6108)
self.assertEqual(res.result.control_statistics.sample_size, 3892)
self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219,
ndecimals)
self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833,
ndecimals)
self.assertAlmostEqual(res.result.statistical_power, 0.36401,
ndecimals)
self.assertEqual(res.result.stop, False)
self.assertEqual(res.test.kpi.name, 'normal_same')
def test_group_sequential_actual_size_larger_than_estimated(self):
""" Check the group sequential function with wrong input,
such that the actual data size is already larger than estimated sample size.
"""
res = es.group_sequential(self.rand_s1, self.rand_s2, estimated_sample_size=100)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
np.testing.assert_almost_equal (value025, -0.24461812530841959, decimal=5)
np.testing.assert_almost_equal (value975, -0.07312917030429833, decimal=5)
def test_find_value_by_key_with_condition(self):
list_of_dicts = [{'bla': 1, 'blu': 2},
{'bla': 3, 'blu': 4},
{'bla': 5, 'blu': 6}]
self.assertEqual(util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5, 'blu'), 6)
with self.assertRaises(IndexError):
util.find_value_by_key_with_condition(list_of_dicts, 'bla', 'value_not_exist', 'blu')
with self.assertRaises(KeyError):
util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5, 'key_not_exist')
def test__delta__2percentiles_no_tests(self):
""" Percentiles of delta() for sga are corrected for no tests (1 as a default). """
res = statx.delta(
self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2])
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
self.assertAlmostEqual(value025, -0.53770569567692295)
self.assertAlmostEqual(value975, -0.040755842784587965)
def test__delta__2percentiles_no_tests(self):
""" Percentiles of delta() for sga are corrected for no tests (1 as a default). """
res = statx.delta(
self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2])
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
self.assertAlmostEqual(value025, -0.53963938185557114)
self.assertAlmostEqual(value975, -0.038822156605939739)
def test__delta__2percentiles_no_tests(self):
""" Percentiles of delta() for sga are corrected for no tests (1 as a default). """
res = statx.delta(self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2])
value025 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 97.5,
'value')
self.assertAlmostEqual(value025, -0.53963938185557114)
self.assertAlmostEqual(value975, -0.038822156605939739)
def test_one_test_in_suite(self):
ndecimals = 5
res = self.getExperiment().analyze_statistical_test_suite(
self.suite_with_one_test)
self.assertEqual(res.correction_method, CorrectionMethod.NONE)
self.assertEqual(len(res.results), 1)
simple_stat_res = res.results[0].result
self.assertAlmostEqual(simple_stat_res.original_test_statistics.delta,
0.033053, ndecimals)
lower_bound_ci = find_value_by_key_with_condition(
simple_stat_res.original_test_statistics.confidence_interval,
'percentile', 2.5, 'value')
upper_bound_ci = find_value_by_key_with_condition(
simple_stat_res.original_test_statistics.confidence_interval,
'percentile', 97.5, 'value')
self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals)
self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals)
self.assertEqual(
simple_stat_res.original_test_statistics.treatment_statistics.
sample_size, 6108)
self.assertEqual(
simple_stat_res.original_test_statistics.control_statistics.
sample_size, 3892)
self.assertAlmostEqual(
simple_stat_res.original_test_statistics.treatment_statistics.mean,
0.025219, ndecimals)
self.assertAlmostEqual(
simple_stat_res.original_test_statistics.control_statistics.mean,
-0.007833, ndecimals)
self.assertAlmostEqual(
simple_stat_res.original_test_statistics.statistical_power,
0.36401, ndecimals)
# corrected test statistics should be the same as original test statistics (checks some values)
self.assertAlmostEqual(simple_stat_res.corrected_test_statistics.delta,
0.033053, ndecimals)
self.assertAlmostEqual(
simple_stat_res.corrected_test_statistics.statistical_power,
0.36401, ndecimals)
self.assertEqual(
simple_stat_res.corrected_test_statistics.treatment_statistics.
sample_size, 6108)
self.assertEqual(
simple_stat_res.corrected_test_statistics.control_statistics.
sample_size, 3892)
def test_bayes_precision_delta(self):
ndecimals = 5
res = self.getExperiment().analyze_statistical_test(self.test_normal_same,
test_method='bayes_precision',
num_iters=2000)
self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals)
lower_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 2.5, 'value')
upper_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 97.5, 'value')
self.assertAlmostEqual(lower_bound_ci, -0.00829, ndecimals)
self.assertAlmostEqual(upper_bound_ci, 0.07127, ndecimals)
self.assertEqual(res.result.treatment_statistics.sample_size, 6108)
self.assertEqual(res.result.control_statistics.sample_size, 3892)
self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219, ndecimals)
self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833, ndecimals)
self.assertEqual(res.result.stop, True)
def test_bayes_factor(self):
""" Check the Bayes factor function."""
res = es.bayes_factor(self.rand_s1, self.rand_s2, num_iters=2000)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean, -0.045256707490195384)
self.assertAlmostEqual(res.control_statistics.mean, 0.11361694031616358)
self.assertAlmostEqual(res.treatment_statistics.variance, 0.9742344563121542)
self.assertAlmostEqual(res.control_statistics.variance, 0.9373337542827797)
self.assertAlmostEqual(res.delta, -0.15887364780635896)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
np.testing.assert_almost_equal(value025, -0.24293384641452503, decimal=5)
np.testing.assert_almost_equal(value975, -0.075064346336461404, decimal=5)
self.assertEqual(res.p, None)
self.assertEqual(res.statistical_power, None)
self.assertEqual(res.stop, True)
def test_bayes_factor_poisson(self):
""" Check the Bayes factor function for Poisson distributions."""
res = es.bayes_factor(self.rand_s3, self.rand_s4, distribution='poisson', num_iters=2000)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean, 0.96599999999999997)
self.assertAlmostEqual(res.control_statistics.mean, 2.9249999999999998)
self.assertAlmostEqual(res.treatment_statistics.variance, 0.868844)
self.assertAlmostEqual(res.control_statistics.variance, 2.901375)
self.assertAlmostEqual(res.delta, -1.9589999999999999)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
np.testing.assert_almost_equal(value025, -2.0713281392132465, decimal=5)
np.testing.assert_almost_equal(value975, -1.8279692168150592, decimal=5)
self.assertEqual(res.p, None)
self.assertEqual(res.statistical_power, None)
self.assertEqual(res.stop, True)
def test_group_sequential_actual_size_larger_than_estimated(self):
""" Check the group sequential function with wrong input,
such that the actual data size is already larger than estimated sample size.
"""
res = es.group_sequential(self.rand_s1,
self.rand_s2,
estimated_sample_size=100)
value025 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 97.5,
'value')
np.testing.assert_almost_equal(value025,
-0.24461812530841959,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.07312917030429833,
decimal=5)
def test_fixed_horizon(self):
ndecimals = 5
res = self.getExperiment().analyze_statistical_test(self.test_normal_same, test_method='fixed_horizon')
self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals)
lower_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 2.5, 'value')
upper_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 97.5, 'value')
self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals)
self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals)
self.assertEqual(res.result.treatment_statistics.sample_size, 6108)
self.assertEqual(res.result.control_statistics.sample_size, 3892)
self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219, ndecimals)
self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833, ndecimals)
self.assertAlmostEqual(res.result.statistical_power, 0.36401, ndecimals)
self.assertEqual(res.test.kpi.name, 'normal_same')
def test_group_sequential(self):
""" Check the group sequential function."""
res = es.group_sequential(self.rand_s1, self.rand_s2)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean, -0.045256707490195384)
self.assertAlmostEqual(res.control_statistics.mean, 0.11361694031616358)
self.assertAlmostEqual(res.treatment_statistics.variance, 0.9742344563121542)
self.assertAlmostEqual(res.control_statistics.variance, 0.9373337542827797)
self.assertAlmostEqual(res.delta, -0.15887364780635896)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
np.testing.assert_almost_equal(value025, -0.24461812530841959, decimal=5)
np.testing.assert_almost_equal(value975, -0.07312917030429833, decimal=5)
self.assertAlmostEqual(res.p, 0.0002863669955157941)
self.assertAlmostEqual(res.statistical_power, 0.9529152504960496)
self.assertEqual(res.stop, True)
def test__delta__computation_not_assumed_normal(self):
""" Result of delta() not assuming normality equals expected result. """
# Computing delta not assumed normal
res = statx.delta(
self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2],
assume_normal=True)
# Checking if mean has right value
self.assertAlmostEqual(res.delta, -0.28923076923075541)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
# Checking if lower percentile has right value
self.assertAlmostEqual(value025, -0.53770569567692295)
# Checking if uper percentile has right value
self.assertAlmostEqual(value975, -0.040755842784587965)
# Checking if sample size 1 is correct
self.assertEqual(res.treatment_statistics.sample_size, 65)
# Checking if sample size 2 is correct
self.assertEqual(res.control_statistics.sample_size, 65)
def test__delta__computation_not_assumed_normal(self):
""" Result of delta() not assuming normality equals expected result. """
# Computing delta not assumed normal
res = statx.delta(
self.samples.temperature[self.samples.gender == 1],
self.samples.temperature[self.samples.gender == 2],
assume_normal=True)
# Checking if mean has right value
self.assertAlmostEqual(res.delta, -0.28923076923075541)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
# Checking if lower percentile has right value
self.assertAlmostEqual(value025, -0.53963938185557114)
# Checking if uper percentile has right value
self.assertAlmostEqual(value975, -0.038822156605939739)
# Checking if sample size 1 is correct
self.assertEqual(res.treatment_statistics.sample_size, 65)
# Checking if sample size 2 is correct
self.assertEqual(res.control_statistics.sample_size, 65)
def test_find_value_by_key_with_condition(self):
list_of_dicts = [{
'bla': 1,
'blu': 2
}, {
'bla': 3,
'blu': 4
}, {
'bla': 5,
'blu': 6
}]
self.assertEqual(
util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5,
'blu'), 6)
with self.assertRaises(IndexError):
util.find_value_by_key_with_condition(list_of_dicts, 'bla',
'value_not_exist', 'blu')
with self.assertRaises(KeyError):
util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5,
'key_not_exist')
def test_bayes_precision_delta(self):
ndecimals = 5
res = self.getExperiment().analyze_statistical_test(
self.test_normal_same,
test_method='bayes_precision',
num_iters=2000)
self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals)
lower_bound_ci = find_value_by_key_with_condition(
res.result.confidence_interval, 'percentile', 2.5, 'value')
upper_bound_ci = find_value_by_key_with_condition(
res.result.confidence_interval, 'percentile', 97.5, 'value')
self.assertAlmostEqual(lower_bound_ci, -0.00829, ndecimals)
self.assertAlmostEqual(upper_bound_ci, 0.07127, ndecimals)
self.assertEqual(res.result.treatment_statistics.sample_size, 6108)
self.assertEqual(res.result.control_statistics.sample_size, 3892)
self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219,
ndecimals)
self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833,
ndecimals)
self.assertEqual(res.result.stop, True)
