def test_bayes_precision_delta(self): ndecimals = 5 res = self.getExperiment(['normal_same' ]).delta(method='bayes_precision', num_iters=2000) variants = find_list_of_dicts_element(res['kpis'], 'name', 'normal_same', 'variants') aStats = find_list_of_dicts_element(variants, 'name', 'A', 'delta_statistics') self.assertNumericalEqual(aStats['delta'], 0.033053, ndecimals) self.assertEqual(aStats['stop'], True, ndecimals) self.assertEqual(aStats['number_of_iterations'], 2000, ndecimals) self.assertNumericalEqual(aStats['confidence_interval'][0]['value'], -0.00829, ndecimals) self.assertNumericalEqual(aStats['confidence_interval'][1]['value'], 0.07127, ndecimals) self.assertEqual(aStats['treatment_sample_size'], 6108) self.assertEqual(aStats['control_sample_size'], 3892) self.assertNumericalEqual(aStats['treatment_mean'], 0.025219, ndecimals) self.assertNumericalEqual(aStats['control_mean'], -0.007833, ndecimals) self.assertNumericalEqual(aStats['statistical_power'], 0.36401, ndecimals)
def test__delta__computation_assumed_normal(self): """ Result of delta() assuming normality equals expected result. """ # Computing delta assumed normal res = statx.delta(self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2], percentiles=[2.5, 97.5], assume_normal=True) # Checking if mean has right value self.assertAlmostEqual(res['delta'], -0.28923076923075541) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value') # Checking if lower percentile has right value self.assertAlmostEqual(value025, -0.53770569567692295) # Checking if uper percentile has right value self.assertAlmostEqual(value975, -0.040755842784587965) # Checking if sample size 1 is correct self.assertEqual(res['treatment_sample_size'], 65) # Checking if sample size 2 is correct self.assertEqual(res['control_sample_size'], 65)
def test_group_sequential_delta(self): ndecimals = 5 res = self.getExperiment(['normal_same' ]).delta(method='group_sequential') variants = find_list_of_dicts_element(res['kpis'], 'name', 'normal_same', 'variants') aStats = find_list_of_dicts_element(variants, 'name', 'A', 'delta_statistics') self.assertNumericalEqual(aStats['delta'], 0.033053, ndecimals) self.assertNumericalEqual(aStats['confidence_interval'][0]['value'], -0.007135, ndecimals) self.assertNumericalEqual(aStats['confidence_interval'][1]['value'], 0.073240, ndecimals) self.assertEqual(aStats['treatment_sample_size'], 6108) self.assertEqual(aStats['control_sample_size'], 3892) self.assertNumericalEqual(aStats['treatment_mean'], 0.025219, ndecimals) self.assertNumericalEqual(aStats['control_mean'], -0.007833, ndecimals) self.assertNumericalEqual(aStats['statistical_power'], 0.36401, ndecimals)
def test_group_sequential_multi_test_correction(self): """ Test group sequential with multiple correction """ res = es.group_sequential(self.rand_s1, self.rand_s2, multi_test_correction=True, num_tests=25) self.assertEqual(res['stop'], True) self.assertAlmostEqual(res['delta'], -0.15887364780635896) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 0.1, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 99.9, 'value') np.testing.assert_almost_equal(value025, -0.29416175390519078, decimal=5) np.testing.assert_almost_equal(value975, -0.023585541707525692, decimal=5) self.assertEqual(res['treatment_sample_size'], 1000) self.assertEqual(res['control_sample_size'], 1000) self.assertAlmostEqual(res['treatment_mean'], -0.045256707490195384) self.assertAlmostEqual(res['control_mean'], 0.11361694031616358)
def test_bayes_factor_poisson(self): """ Check the Bayes factor function for Poisson distributions. """ res= es.bayes_factor(self.rand_s3, self.rand_s4, distribution='poisson', num_iters=2000) self.assertEqual (res['stop'], True) self.assertAlmostEqual (res['delta'], -1.9589999999999999) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value') self.assertAlmostEqual (value025, -2.0713281392132465) self.assertAlmostEqual (value975, -1.8279692168150592) self.assertEqual (res['treatment_sample_size'], 1000) self.assertEqual (res['control_sample_size'], 1000) self.assertAlmostEqual (res['treatment_mean'], 0.96599999999999997) self.assertAlmostEqual (res['control_mean'], 2.9249999999999998)
def test_bayes_factor(self): """ Check the Bayes factor function. """ res = es.bayes_factor(self.rand_s1, self.rand_s2, num_iters=2000) self.assertEqual (res['stop'], True) self.assertAlmostEqual (res['delta'], -0.15887364780635896) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value') self.assertAlmostEqual (value025, -0.24293384641452503) self.assertAlmostEqual (value975, -0.075064346336461404) self.assertEqual (res['treatment_sample_size'], 1000) self.assertEqual (res['control_sample_size'], 1000) self.assertAlmostEqual (res['treatment_mean'], -0.045256707490195384) self.assertAlmostEqual (res['control_mean'], 0.11361694031616358)
def test_group_sequential(self): """ Check the group sequential function. """ res = es.group_sequential(self.rand_s1, self.rand_s2) self.assertEqual (res['stop'], True) self.assertAlmostEqual (res['delta'], -0.15887364780635896) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value') np.testing.assert_almost_equal (value025, -0.24461812530841959, decimal=5) np.testing.assert_almost_equal (value975, -0.07312917030429833, decimal=5) self.assertEqual (res['treatment_sample_size'], 1000) self.assertEqual (res['control_sample_size'], 1000) self.assertAlmostEqual (res['treatment_mean'], -0.045256707490195384) self.assertAlmostEqual (res['control_mean'], 0.11361694031616358)
def test__delta__2percentiles_no_tests(self): """ Percentiles of delta() for sga are corrected for no tests (1 as a default) """ res = statx.delta(self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2], percentiles=[2.5, 97.5], assume_normal=True, num_tests=1) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value') self.assertAlmostEqual(value025, -0.53770569567692295) self.assertAlmostEqual(value975, -0.040755842784587965)
def test__delta__2percentiles_25_tests(self): """ Percentiles of delta() for sga are corrected for 25 tests """ res = statx.delta(self.samples.temperature[self.samples.gender == 1], self.samples.temperature[self.samples.gender == 2], percentiles=[2.5, 97.5], assume_normal=True, num_tests=25) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 0.1, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 99.9, 'value') self.assertAlmostEqual(value025, -0.68544085117601006) self.assertAlmostEqual(value975, 0.1069793127145035)
def test_sga_date(self): exp = self.getExperiment([self.derived_kpi_1['name']], [self.derived_kpi_1]) sga_result = exp.sga_date() self.assertEqual(len(sga_result), 417) numerical_dimension_name = find_list_of_dicts_element( sga_result, "segment", "['2016-01-21']", "dimension") self.assertEqual(numerical_dimension_name, 'date')
def test_group_sequential_actual_size_larger_than_estimated(self): """ Check the group sequential function with wrong input, such that the actual data size is already larger than estimated sample size. """ res = es.group_sequential(self.rand_s1, self.rand_s2, estimated_sample_size=100) value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value') value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value') np.testing.assert_almost_equal(value025, -0.24461812530841959, decimal=5) np.testing.assert_almost_equal(value975, -0.07312917030429833, decimal=5)
def test_sga_multi_dim_multi_bins(self): exp = self.getExperiment([self.derived_kpi_1['name']], [self.derived_kpi_1]) dimension_to_bin = { "normal_same": [ Bin("numerical", 1, 2, True, False), Bin("numerical", 2, 3, True, False) ], "feature": [Bin("categorical", ["has"]), Bin("categorical", ["non"])] } sga_result = exp.sga(dimension_to_bin) self.assertEqual(len(sga_result), 4) numerical_dimension_name = find_list_of_dicts_element( sga_result, "segment", "[1, 2)", "dimension") self.assertEqual(numerical_dimension_name, 'normal_same') categorical_dimension_name = find_list_of_dicts_element( sga_result, "segment", "['non']", "dimension") self.assertEqual(categorical_dimension_name, 'feature')
def test_bayes_precision_delta(self): ndecimals = 5 res = self.getExperiment(['normal_same']).delta(method='bayes_precision', num_iters=2000) variants = find_list_of_dicts_element(res['kpis'], 'name', 'normal_same', 'variants') aStats = find_list_of_dicts_element(variants, 'name', 'A', 'delta_statistics') self.assertNumericalEqual(aStats['delta'], 0.033053, ndecimals) self.assertEqual(aStats['stop'], True, ndecimals) self.assertEqual(aStats['number_of_iterations'], 2000, ndecimals) # # this can result in different numerical values depending on Python version # # self.assertNumericalEqual(aStats['confidence_interval'][0]['value'], -0.007079081, ndecimals) # self.assertNumericalEqual(aStats['confidence_interval'][1]['value'], 0.072703576, ndecimals) self.assertEqual(aStats['treatment_sample_size'], 6108) self.assertEqual(aStats['control_sample_size'], 3892) self.assertNumericalEqual(aStats['treatment_mean'], 0.025219, ndecimals) self.assertNumericalEqual(aStats['control_mean'], -0.007833, ndecimals)
def test_find_dict_element(self): list_of_dicts = [{ 'bla': 1, 'blu': 2 }, { 'bla': 3, 'blu': 4 }, { 'bla': 5, 'blu': 6 }] self.assertEqual( util.find_list_of_dicts_element(list_of_dicts, 'bla', 5, 'blu'), 6)