def test_exploit_variant_1_single_trial_still_exploring(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
seen = collections.Counter()
mab.trial(test_name, 'variant1') # first trial with no success
for x in range(1000):
bucket = mab.get_bucket(test_name, buckets)
seen[bucket] += 1
self.assertTrue(bucket in buckets)
self.assertGreaterEqual(seen['control'], 275)
self.assertGreaterEqual(seen['variant1'], 275)
self.assertGreaterEqual(seen['variant2'], 275)
self.assertEqual(sum(seen.values()), 1000)
def test_variant_dominates_low_exploration(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
seen = collections.Counter()
for x in range(3): # poor showing, control...
mab.trial(test_name, 'control')
mab.success(test_name, 'control')
for x in range(9): # Not bad, variant1...
mab.trial(test_name, 'variant2')
mab.success(test_name, 'variant2')
for x in range(100): # Variant2 leading the way.
mab.trial(test_name, 'variant2')
mab.success(test_name, 'variant2')
for x in range(1000):
bucket = mab.get_bucket(test_name, buckets, rate=.05)
seen[bucket] += 1
self.assertTrue(bucket in buckets)
self.assertGreaterEqual(seen['control'], 5)
self.assertLessEqual(seen['control'], 75)
self.assertGreaterEqual(seen['variant1'], 5)
self.assertLessEqual(seen['variant1'], 75)
self.assertGreaterEqual(seen['variant2'], 900)
self.assertLessEqual(seen['variant2'], 990)
self.assertEqual(sum(seen.values()), 1000)
def test_get_state_with_tuples_used(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
test_name = 'my_test_v1'
buckets = ['control', 'variant1', 'variant2']
mab.trial('my_test_v1', 'control')
mab.trial('my_test_v1', 'control')
mab.success('my_test_v1', 'control')
mab.success('my_test_v1', 'control')
mab.trial('my_test_v1', 'variant1')
mab.success('my_test_v1', 'variant1')
mab.trial('my_test_v1', 'variant2')
data = mab.get_state_with_tuples(test_name, buckets)
self.assertEqual(list(data.keys()), buckets)
self.assertEqual(data['control'][0], 2)
self.assertEqual(data['control'][1], 2)
self.assertEqual(data['variant1'][0], 1)
self.assertEqual(data['variant1'][1], 1)
self.assertEqual(data['variant2'][0], 1)
self.assertEqual(data['variant2'][1], 0)
def test_trial(self):
with mock.patch('ab.store.get') as store_patch:
store_patch.return_value = store_mock.MockRedis()
self.assertEqual(mab.trial('my_test_v1', 'control'), 1)
self.assertEqual(mab.trial('my_test_v1', 'control'), 2)