def test_simulate_experiment_variant1_outperforms(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
# Trying to simulate random 30% trial, random 10% success rate.
for x in range(10000):
if random.random() <= .3: # ~30% in trial
bucket = mab.get_bucket(test_name, buckets)
# Variant 1 outperforming at ~30% success rate
success_chance = random.random()
if bucket == 'variant1' and success_chance <= .3:
mab.success(test_name, bucket)
# Only 5% of control and variant 2 members succeed.
elif success_chance <= .05:
mab.success(test_name, bucket)
scores = mab.get_scores(test_name, buckets)
self.assertGreaterEqual(scores['control'], .0)
self.assertLessEqual(scores['control'], .15)
# Variant 1 consistently does better.
self.assertGreaterEqual(scores['variant1'], .15)
self.assertLessEqual(scores['variant1'], .35)
self.assertGreaterEqual(scores['variant2'], .0)
self.assertLessEqual(scores['variant2'], .15)
state = mab.get_state_with_tuples(test_name, buckets)
results = stats.get_results(state)
self.assertTrue(results['variant1'].is_better_than_control)
self.assertTrue(results['variant1'].is_significant)
self.assertGreaterEqual(results['variant1'].p_value, 0.0)
self.assertLessEqual(results['variant1'].p_value, 0.001)
def test_simulate_experiment(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
# Trying to simulate random 30% trial, random 10% success rate.
for x in range(10000):
# ~30% in trial
if random.random() <= .3:
bucket = mab.get_bucket(test_name, buckets)
# ~10% of trial members succeed.
if random.random() <= .1:
mab.success(test_name, bucket)
scores = mab.get_scores(test_name, buckets)
# Leaving headroom for random
self.assertLessEqual(scores['control'], .15)
self.assertGreaterEqual(scores['control'], .01)
self.assertLessEqual(scores['variant1'], .15)
self.assertGreaterEqual(scores['variant1'], .01)
self.assertLessEqual(scores['variant2'], .15)
self.assertGreaterEqual(scores['variant2'], .01)
state = mab.get_state_with_tuples(test_name, buckets)
results = stats.get_results(state)
self.assertGreaterEqual(results['variant1'].p_value, 0.0)
self.assertLessEqual(results['variant1'].p_value, 0.5)
def test_variant_dominates_low_exploration(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
seen = collections.Counter()
for x in range(3): # poor showing, control...
mab.trial(test_name, 'control')
mab.success(test_name, 'control')
for x in range(9): # Not bad, variant1...
mab.trial(test_name, 'variant2')
mab.success(test_name, 'variant2')
for x in range(100): # Variant2 leading the way.
mab.trial(test_name, 'variant2')
mab.success(test_name, 'variant2')
for x in range(1000):
bucket = mab.get_bucket(test_name, buckets, rate=.05)
seen[bucket] += 1
self.assertTrue(bucket in buckets)
self.assertGreaterEqual(seen['control'], 5)
self.assertLessEqual(seen['control'], 75)
self.assertGreaterEqual(seen['variant1'], 5)
self.assertLessEqual(seen['variant1'], 75)
self.assertGreaterEqual(seen['variant2'], 900)
self.assertLessEqual(seen['variant2'], 990)
self.assertEqual(sum(seen.values()), 1000)
def test_cold_start_exploit_explores(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
seen = collections.Counter()
for x in range(1000):
bucket = mab.get_bucket(test_name, buckets)
seen[bucket] += 1
self.assertTrue(bucket in buckets)
self.assertGreaterEqual(seen['control'], 275)
self.assertGreaterEqual(seen['variant1'], 275)
self.assertGreaterEqual(seen['variant2'], 275)
self.assertEqual(sum(seen.values()), 1000)
def test_exploit_variant_1_single_trial_still_exploring(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
seen = collections.Counter()
mab.trial(test_name, 'variant1') # first trial with no success
for x in range(1000):
bucket = mab.get_bucket(test_name, buckets)
seen[bucket] += 1
self.assertTrue(bucket in buckets)
self.assertGreaterEqual(seen['control'], 275)
self.assertGreaterEqual(seen['variant1'], 275)
self.assertGreaterEqual(seen['variant2'], 275)
self.assertEqual(sum(seen.values()), 1000)
def home():
buckets = ('a', 'b', 'c')
# TODO: run stats.get_results to compute test results!
scores = mab.get_scores(test='test_name', buckets=buckets)
bucket = mab.get_bucket(test='test_name', buckets=buckets)
return TEMPLATE.format(bucket=bucket, scores=scores)
from ab import mab
TEST_NAME = 'MY_DEMO_TEST_V1'
TEST_CONTROL = 'control'
TEST_VARIANT1 = 'variant1'
TEST_VARIANT2 = 'variant2'
TEST_BUCKETS = (
TEST_CONTROL,
TEST_VARIANT1,
TEST_VARIANT2,
)
while True:
bucket = mab.get_bucket(TEST_NAME, TEST_BUCKETS)
try:
val = input(f'Did {bucket} succeed? y/n: ').strip()
except KeyboardInterrupt:
exit()
if val == 'y':
mab.success(TEST_NAME, bucket)
print(mab.get_scores(TEST_NAME, TEST_BUCKETS))
