def test_bayes_factor_poisson(self): """ Check the Bayes factor function for Poisson distributions.""" res = es.bayes_factor(self.rand_s3, self.rand_s4, distribution='poisson', num_iters=2000) self.assertEqual(res.treatment_statistics.sample_size, 1000) self.assertEqual(res.control_statistics.sample_size, 1000) self.assertAlmostEqual(res.treatment_statistics.mean, 0.96599999999999997) self.assertAlmostEqual(res.control_statistics.mean, 2.9249999999999998) self.assertAlmostEqual(res.treatment_statistics.variance, 0.868844) self.assertAlmostEqual(res.control_statistics.variance, 2.901375) self.assertAlmostEqual(res.delta, -1.9589999999999999) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -2.0713281392132465, decimal=5) np.testing.assert_almost_equal(value975, -1.8279692168150592, decimal=5) self.assertEqual(res.p, None) self.assertEqual(res.statistical_power, None) self.assertEqual(res.stop, True)
def test_one_test_in_suite(self): ndecimals = 5 res = self.getExperiment().analyze_statistical_test_suite(self.suite_with_one_test) self.assertEqual(res.correction_method, CorrectionMethod.NONE) self.assertEqual(len(res.results), 1) simple_stat_res = res.results[0].result self.assertAlmostEqual(simple_stat_res.original_test_statistics.delta, 0.033053, ndecimals) lower_bound_ci = find_value_by_key_with_condition(simple_stat_res.original_test_statistics.confidence_interval, 'percentile', 2.5, 'value') upper_bound_ci = find_value_by_key_with_condition(simple_stat_res.original_test_statistics.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals) self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals) self.assertEqual(simple_stat_res.original_test_statistics.treatment_statistics.sample_size, 6108) self.assertEqual(simple_stat_res.original_test_statistics.control_statistics.sample_size, 3892) self.assertAlmostEqual(simple_stat_res.original_test_statistics.treatment_statistics.mean, 0.025219, ndecimals) self.assertAlmostEqual(simple_stat_res.original_test_statistics.control_statistics.mean, -0.007833, ndecimals) self.assertAlmostEqual(simple_stat_res.original_test_statistics.statistical_power, 0.36401, ndecimals) # corrected test statistics should be the same as original test statistics (checks some values) self.assertAlmostEqual(simple_stat_res.corrected_test_statistics.delta, 0.033053, ndecimals) self.assertAlmostEqual(simple_stat_res.corrected_test_statistics.statistical_power, 0.36401, ndecimals) self.assertEqual(simple_stat_res.corrected_test_statistics.treatment_statistics.sample_size, 6108) self.assertEqual(simple_stat_res.corrected_test_statistics.control_statistics.sample_size, 3892)
def test_bayes_precision(self): """ Check the bayes_precision function.""" res = es.bayes_precision(self.rand_s1, self.rand_s2, num_iters=2000) self.assertEqual(res.treatment_statistics.sample_size, 1000) self.assertEqual(res.control_statistics.sample_size, 1000) self.assertAlmostEqual(res.treatment_statistics.mean, -0.045256707490195384) self.assertAlmostEqual(res.control_statistics.mean, 0.11361694031616358) self.assertAlmostEqual(res.treatment_statistics.variance, 0.9742344563121542) self.assertAlmostEqual(res.control_statistics.variance, 0.9373337542827797) self.assertAlmostEqual(res.delta, -0.15887364780635896) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -0.24293384641452503, decimal=5) np.testing.assert_almost_equal(value975, -0.07506434633646140, decimal=5) self.assertEqual(res.p, None) self.assertEqual(res.statistical_power, None) self.assertEqual(res.stop, False)
def test_group_sequential(self): """ Check the group sequential function.""" res = es.group_sequential(self.rand_s1, self.rand_s2) self.assertEqual(res.treatment_statistics.sample_size, 1000) self.assertEqual(res.control_statistics.sample_size, 1000) self.assertAlmostEqual(res.treatment_statistics.mean, -0.045256707490195384) self.assertAlmostEqual(res.control_statistics.mean, 0.11361694031616358) self.assertAlmostEqual(res.treatment_statistics.variance, 0.9742344563121542) self.assertAlmostEqual(res.control_statistics.variance, 0.9373337542827797) self.assertAlmostEqual(res.delta, -0.15887364780635896) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -0.24461812530841959, decimal=5) np.testing.assert_almost_equal(value975, -0.07312917030429833, decimal=5) self.assertAlmostEqual(res.p, 0.0002863669955157941) self.assertAlmostEqual(res.statistical_power, 0.9529152504960496) self.assertEqual(res.stop, True)
def test_group_sequential(self): ndecimals = 5 res = self.getExperiment().analyze_statistical_test( self.test_normal_same, test_method='group_sequential') self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals) lower_bound_ci = find_value_by_key_with_condition( res.result.confidence_interval, 'percentile', 2.5, 'value') upper_bound_ci = find_value_by_key_with_condition( res.result.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals) self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals) self.assertEqual(res.result.treatment_statistics.sample_size, 6108) self.assertEqual(res.result.control_statistics.sample_size, 3892) self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219, ndecimals) self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833, ndecimals) self.assertAlmostEqual(res.result.statistical_power, 0.36401, ndecimals) self.assertEqual(res.result.stop, False) self.assertEqual(res.test.kpi.name, 'normal_same')
def test_group_sequential_actual_size_larger_than_estimated(self): """ Check the group sequential function with wrong input, such that the actual data size is already larger than estimated sample size. """ res = es.group_sequential(self.rand_s1, self.rand_s2, estimated_sample_size=100) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal (value025, -0.24461812530841959, decimal=5) np.testing.assert_almost_equal (value975, -0.07312917030429833, decimal=5)
def test_find_value_by_key_with_condition(self): list_of_dicts = [{'bla': 1, 'blu': 2}, {'bla': 3, 'blu': 4}, {'bla': 5, 'blu': 6}] self.assertEqual(util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5, 'blu'), 6) with self.assertRaises(IndexError): util.find_value_by_key_with_condition(list_of_dicts, 'bla', 'value_not_exist', 'blu') with self.assertRaises(KeyError): util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5, 'key_not_exist')
def test__delta__2percentiles_no_tests(self): """ Percentiles of delta() for sga are corrected for no tests (1 as a default). """ res = statx.delta( self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2]) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(value025, -0.53770569567692295) self.assertAlmostEqual(value975, -0.040755842784587965)
def test__delta__2percentiles_no_tests(self): """ Percentiles of delta() for sga are corrected for no tests (1 as a default). """ res = statx.delta( self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2]) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(value025, -0.53963938185557114) self.assertAlmostEqual(value975, -0.038822156605939739)
def test__delta__2percentiles_no_tests(self): """ Percentiles of delta() for sga are corrected for no tests (1 as a default). """ res = statx.delta(self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2]) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(value025, -0.53963938185557114) self.assertAlmostEqual(value975, -0.038822156605939739)
def test_one_test_in_suite(self): ndecimals = 5 res = self.getExperiment().analyze_statistical_test_suite( self.suite_with_one_test) self.assertEqual(res.correction_method, CorrectionMethod.NONE) self.assertEqual(len(res.results), 1) simple_stat_res = res.results[0].result self.assertAlmostEqual(simple_stat_res.original_test_statistics.delta, 0.033053, ndecimals) lower_bound_ci = find_value_by_key_with_condition( simple_stat_res.original_test_statistics.confidence_interval, 'percentile', 2.5, 'value') upper_bound_ci = find_value_by_key_with_condition( simple_stat_res.original_test_statistics.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals) self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals) self.assertEqual( simple_stat_res.original_test_statistics.treatment_statistics. sample_size, 6108) self.assertEqual( simple_stat_res.original_test_statistics.control_statistics. sample_size, 3892) self.assertAlmostEqual( simple_stat_res.original_test_statistics.treatment_statistics.mean, 0.025219, ndecimals) self.assertAlmostEqual( simple_stat_res.original_test_statistics.control_statistics.mean, -0.007833, ndecimals) self.assertAlmostEqual( simple_stat_res.original_test_statistics.statistical_power, 0.36401, ndecimals) # corrected test statistics should be the same as original test statistics (checks some values) self.assertAlmostEqual(simple_stat_res.corrected_test_statistics.delta, 0.033053, ndecimals) self.assertAlmostEqual( simple_stat_res.corrected_test_statistics.statistical_power, 0.36401, ndecimals) self.assertEqual( simple_stat_res.corrected_test_statistics.treatment_statistics. sample_size, 6108) self.assertEqual( simple_stat_res.corrected_test_statistics.control_statistics. sample_size, 3892)
def test_bayes_precision_delta(self): ndecimals = 5 res = self.getExperiment().analyze_statistical_test(self.test_normal_same, test_method='bayes_precision', num_iters=2000) self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals) lower_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 2.5, 'value') upper_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(lower_bound_ci, -0.00829, ndecimals) self.assertAlmostEqual(upper_bound_ci, 0.07127, ndecimals) self.assertEqual(res.result.treatment_statistics.sample_size, 6108) self.assertEqual(res.result.control_statistics.sample_size, 3892) self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219, ndecimals) self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833, ndecimals) self.assertEqual(res.result.stop, True)
def test_bayes_factor(self): """ Check the Bayes factor function.""" res = es.bayes_factor(self.rand_s1, self.rand_s2, num_iters=2000) self.assertEqual(res.treatment_statistics.sample_size, 1000) self.assertEqual(res.control_statistics.sample_size, 1000) self.assertAlmostEqual(res.treatment_statistics.mean, -0.045256707490195384) self.assertAlmostEqual(res.control_statistics.mean, 0.11361694031616358) self.assertAlmostEqual(res.treatment_statistics.variance, 0.9742344563121542) self.assertAlmostEqual(res.control_statistics.variance, 0.9373337542827797) self.assertAlmostEqual(res.delta, -0.15887364780635896) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -0.24293384641452503, decimal=5) np.testing.assert_almost_equal(value975, -0.075064346336461404, decimal=5) self.assertEqual(res.p, None) self.assertEqual(res.statistical_power, None) self.assertEqual(res.stop, True)
def test_bayes_factor_poisson(self): """ Check the Bayes factor function for Poisson distributions.""" res = es.bayes_factor(self.rand_s3, self.rand_s4, distribution='poisson', num_iters=2000) self.assertEqual(res.treatment_statistics.sample_size, 1000) self.assertEqual(res.control_statistics.sample_size, 1000) self.assertAlmostEqual(res.treatment_statistics.mean, 0.96599999999999997) self.assertAlmostEqual(res.control_statistics.mean, 2.9249999999999998) self.assertAlmostEqual(res.treatment_statistics.variance, 0.868844) self.assertAlmostEqual(res.control_statistics.variance, 2.901375) self.assertAlmostEqual(res.delta, -1.9589999999999999) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -2.0713281392132465, decimal=5) np.testing.assert_almost_equal(value975, -1.8279692168150592, decimal=5) self.assertEqual(res.p, None) self.assertEqual(res.statistical_power, None) self.assertEqual(res.stop, True)
def test_group_sequential_actual_size_larger_than_estimated(self): """ Check the group sequential function with wrong input, such that the actual data size is already larger than estimated sample size. """ res = es.group_sequential(self.rand_s1, self.rand_s2, estimated_sample_size=100) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -0.24461812530841959, decimal=5) np.testing.assert_almost_equal(value975, -0.07312917030429833, decimal=5)
def test_fixed_horizon(self): ndecimals = 5 res = self.getExperiment().analyze_statistical_test(self.test_normal_same, test_method='fixed_horizon') self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals) lower_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 2.5, 'value') upper_bound_ci = find_value_by_key_with_condition(res.result.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(lower_bound_ci, -0.007135, ndecimals) self.assertAlmostEqual(upper_bound_ci, 0.073240, ndecimals) self.assertEqual(res.result.treatment_statistics.sample_size, 6108) self.assertEqual(res.result.control_statistics.sample_size, 3892) self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219, ndecimals) self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833, ndecimals) self.assertAlmostEqual(res.result.statistical_power, 0.36401, ndecimals) self.assertEqual(res.test.kpi.name, 'normal_same')
def test_group_sequential(self): """ Check the group sequential function.""" res = es.group_sequential(self.rand_s1, self.rand_s2) self.assertEqual(res.treatment_statistics.sample_size, 1000) self.assertEqual(res.control_statistics.sample_size, 1000) self.assertAlmostEqual(res.treatment_statistics.mean, -0.045256707490195384) self.assertAlmostEqual(res.control_statistics.mean, 0.11361694031616358) self.assertAlmostEqual(res.treatment_statistics.variance, 0.9742344563121542) self.assertAlmostEqual(res.control_statistics.variance, 0.9373337542827797) self.assertAlmostEqual(res.delta, -0.15887364780635896) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -0.24461812530841959, decimal=5) np.testing.assert_almost_equal(value975, -0.07312917030429833, decimal=5) self.assertAlmostEqual(res.p, 0.0002863669955157941) self.assertAlmostEqual(res.statistical_power, 0.9529152504960496) self.assertEqual(res.stop, True)
def test__delta__computation_not_assumed_normal(self): """ Result of delta() not assuming normality equals expected result. """ # Computing delta not assumed normal res = statx.delta( self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2], assume_normal=True) # Checking if mean has right value self.assertAlmostEqual(res.delta, -0.28923076923075541) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') # Checking if lower percentile has right value self.assertAlmostEqual(value025, -0.53770569567692295) # Checking if uper percentile has right value self.assertAlmostEqual(value975, -0.040755842784587965) # Checking if sample size 1 is correct self.assertEqual(res.treatment_statistics.sample_size, 65) # Checking if sample size 2 is correct self.assertEqual(res.control_statistics.sample_size, 65)
def test__delta__computation_not_assumed_normal(self): """ Result of delta() not assuming normality equals expected result. """ # Computing delta not assumed normal res = statx.delta( self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2], assume_normal=True) # Checking if mean has right value self.assertAlmostEqual(res.delta, -0.28923076923075541) value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value') value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value') # Checking if lower percentile has right value self.assertAlmostEqual(value025, -0.53963938185557114) # Checking if uper percentile has right value self.assertAlmostEqual(value975, -0.038822156605939739) # Checking if sample size 1 is correct self.assertEqual(res.treatment_statistics.sample_size, 65) # Checking if sample size 2 is correct self.assertEqual(res.control_statistics.sample_size, 65)
def test_find_value_by_key_with_condition(self): list_of_dicts = [{ 'bla': 1, 'blu': 2 }, { 'bla': 3, 'blu': 4 }, { 'bla': 5, 'blu': 6 }] self.assertEqual( util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5, 'blu'), 6) with self.assertRaises(IndexError): util.find_value_by_key_with_condition(list_of_dicts, 'bla', 'value_not_exist', 'blu') with self.assertRaises(KeyError): util.find_value_by_key_with_condition(list_of_dicts, 'bla', 5, 'key_not_exist')
def test_bayes_precision_delta(self): ndecimals = 5 res = self.getExperiment().analyze_statistical_test( self.test_normal_same, test_method='bayes_precision', num_iters=2000) self.assertAlmostEqual(res.result.delta, 0.033053, ndecimals) lower_bound_ci = find_value_by_key_with_condition( res.result.confidence_interval, 'percentile', 2.5, 'value') upper_bound_ci = find_value_by_key_with_condition( res.result.confidence_interval, 'percentile', 97.5, 'value') self.assertAlmostEqual(lower_bound_ci, -0.00829, ndecimals) self.assertAlmostEqual(upper_bound_ci, 0.07127, ndecimals) self.assertEqual(res.result.treatment_statistics.sample_size, 6108) self.assertEqual(res.result.control_statistics.sample_size, 3892) self.assertAlmostEqual(res.result.treatment_statistics.mean, 0.025219, ndecimals) self.assertAlmostEqual(res.result.control_statistics.mean, -0.007833, ndecimals) self.assertEqual(res.result.stop, True)