def test_group_sequential(self):
""" Check the group sequential function."""
res = es.group_sequential(self.rand_s1, self.rand_s2)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean,
-0.045256707490195384)
self.assertAlmostEqual(res.control_statistics.mean,
0.11361694031616358)
self.assertAlmostEqual(res.treatment_statistics.variance,
0.9742344563121542)
self.assertAlmostEqual(res.control_statistics.variance,
0.9373337542827797)
self.assertAlmostEqual(res.delta, -0.15887364780635896)
value025 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 97.5,
'value')
np.testing.assert_almost_equal(value025,
-0.24461812530841959,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.07312917030429833,
decimal=5)
self.assertAlmostEqual(res.p, 0.0002863669955157941)
self.assertAlmostEqual(res.statistical_power, 0.9529152504960496)
self.assertEqual(res.stop, True)
def test_group_sequential_multi_test_correction(self):
"""
Test group sequential with multiple correction
"""
res = es.group_sequential(self.rand_s1,
self.rand_s2,
multi_test_correction=True,
num_tests=25)
self.assertEqual(res['stop'], True)
self.assertAlmostEqual(res['delta'], -0.15887364780635896)
value025 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 0.1, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'],
'percentile', 99.9, 'value')
np.testing.assert_almost_equal(value025,
-0.29416175390519078,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.023585541707525692,
decimal=5)
self.assertEqual(res['treatment_sample_size'], 1000)
self.assertEqual(res['control_sample_size'], 1000)
self.assertAlmostEqual(res['treatment_mean'], -0.045256707490195384)
self.assertAlmostEqual(res['control_mean'], 0.11361694031616358)
def test_group_sequential_actual_size_larger_than_estimated(self):
""" Check the group sequential function with wrong input,
such that the actual data size is already larger than estimated sample size.
"""
res = es.group_sequential(self.rand_s1, self.rand_s2, estimated_sample_size=100)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
np.testing.assert_almost_equal (value025, -0.24461812530841959, decimal=5)
np.testing.assert_almost_equal (value975, -0.07312917030429833, decimal=5)
def group_sequential_delta(self,
result,
kpis_to_analyse,
spending_function='obrien_fleming',
information_fraction=1,
alpha=0.05,
cap=8,
**kwargs):
"""
Calculate the stopping criterion based on the group sequential design
and the effect size.
Args:
result: the initialized Results object
kpis_to_analyse: list of KPIs to be analysed
spending_function: name of the alpha spending function, currently
supports: 'obrien_fleming'
information_fraction: share of the information amount at the point
of evaluation, e.g. the share of the maximum sample size
alpha: type-I error rate
cap: upper bound of the adapted z-score
Returns:
a Results object
"""
if 'estimatedSampleSize' not in self.metadata:
raise ValueError("Missing 'estimatedSampleSize' for the group sequential method!")
for mname in kpis_to_analyse:
metric_df = self.kpis.reset_index()[['entity', 'variant', mname]]
baseline_metric = metric_df.iloc[:, 2][metric_df.iloc[:, 1] == self.baseline_variant]
current_sample_size = float(sum(~metric_df[mname].isnull()))
do_delta = (lambda f: early_stopping_to_dataframe(f.columns[2],
*es.group_sequential(
x=f.iloc[:, 2],
y=baseline_metric,
spending_function=spending_function,
information_fraction=current_sample_size/self.metadata['estimatedSampleSize'],
alpha=alpha,
cap=cap)))
# Actual calculation
df = metric_df.groupby('variant').apply(do_delta).unstack(0)
# force the stop label of the baseline variant to 0
df.loc[(mname,'-',slice(None),'stop'),('value',self.baseline_variant)] = 0
if result.df is None:
result.df = df
else:
result.df = result.df.append(df)
return result
def test_group_sequential(self):
"""
Check the group sequential function.
"""
res = es.group_sequential(self.rand_s1, self.rand_s2)
self.assertEqual (res['stop'], True)
self.assertAlmostEqual (res['delta'], -0.15887364780635896)
value025 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 2.5, 'value')
value975 = find_list_of_dicts_element(res['confidence_interval'], 'percentile', 97.5, 'value')
np.testing.assert_almost_equal (value025, -0.24461812530841959, decimal=5)
np.testing.assert_almost_equal (value975, -0.07312917030429833, decimal=5)
self.assertEqual (res['treatment_sample_size'], 1000)
self.assertEqual (res['control_sample_size'], 1000)
self.assertAlmostEqual (res['treatment_mean'], -0.045256707490195384)
self.assertAlmostEqual (res['control_mean'], 0.11361694031616358)
def test_group_sequential(self):
"""
Check the group sequential function.
"""
res = es.group_sequential(self.rand_s1, self.rand_s2)
self.assertEqual(res['stop'], True)
self.assertAlmostEqual(res['delta'], -0.15887364780635896)
np.testing.assert_almost_equal(res['interval'][02.5],
-0.24461812530841959,
decimal=5)
np.testing.assert_almost_equal(res['interval'][97.5],
-0.07312917030429833,
decimal=5)
self.assertEqual(res['n_x'], 1000)
self.assertEqual(res['n_y'], 1000)
self.assertAlmostEqual(res['mu_x'], -0.045256707490195384)
self.assertAlmostEqual(res['mu_y'], 0.11361694031616358)
def test_group_sequential_actual_size_larger_than_estimated(self):
""" Check the group sequential function with wrong input,
such that the actual data size is already larger than estimated sample size.
"""
res = es.group_sequential(self.rand_s1,
self.rand_s2,
estimated_sample_size=100)
value025 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval,
'percentile', 97.5,
'value')
np.testing.assert_almost_equal(value025,
-0.24461812530841959,
decimal=5)
np.testing.assert_almost_equal(value975,
-0.07312917030429833,
decimal=5)
def test_group_sequential(self):
"""
Check the group sequential function.
"""
stop, delta, CI, n_x, n_y, mu_x, mu_y = es.group_sequential(
self.rand_s1, self.rand_s2)
self.assertEqual(stop, 1)
self.assertAlmostEqual(delta, -0.15887364780635896)
# np.testing.assert_almost_equal(CI.values(), [-0.24461812530841959, -0.07312917030429833], decimal=5)
np.testing.assert_almost_equal(CI[97.5],
-0.07312917030429833,
decimal=5)
np.testing.assert_almost_equal(CI[2.5000000000000022],
-0.24461812530841959,
decimal=5)
self.assertEqual(n_x, 1000)
self.assertEqual(n_y, 1000)
self.assertAlmostEqual(mu_x, -0.045256707490195384)
self.assertAlmostEqual(mu_y, 0.11361694031616358)
def test_group_sequential(self):
""" Check the group sequential function."""
res = es.group_sequential(self.rand_s1, self.rand_s2)
self.assertEqual(res.treatment_statistics.sample_size, 1000)
self.assertEqual(res.control_statistics.sample_size, 1000)
self.assertAlmostEqual(res.treatment_statistics.mean, -0.045256707490195384)
self.assertAlmostEqual(res.control_statistics.mean, 0.11361694031616358)
self.assertAlmostEqual(res.treatment_statistics.variance, 0.9742344563121542)
self.assertAlmostEqual(res.control_statistics.variance, 0.9373337542827797)
self.assertAlmostEqual(res.delta, -0.15887364780635896)
value025 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 2.5, 'value')
value975 = find_value_by_key_with_condition(res.confidence_interval, 'percentile', 97.5, 'value')
np.testing.assert_almost_equal(value025, -0.24461812530841959, decimal=5)
np.testing.assert_almost_equal(value975, -0.07312917030429833, decimal=5)
self.assertAlmostEqual(res.p, 0.0002863669955157941)
self.assertAlmostEqual(res.statistical_power, 0.9529152504960496)
self.assertEqual(res.stop, True)
