def test_invalid_kpi(self):
ret = Experiment.validate_kpi('!hello-jose')
self.assertFalse(ret)
ret = Experiment.validate_kpi('thunder storm')
self.assertFalse(ret)
ret = Experiment.validate_kpi('&!&&!&')
self.assertFalse(ret)
def test_valid_traffic_fractions_save(self):
# test the hidden prop gets set
exp = Experiment.find_or_create('dist-02', ['dist', '100'], traffic_fraction=0.02, redis=self.redis)
self.assertEqual(exp._traffic_fraction, 0.02)
exp = Experiment.find_or_create('dist-100', ['dist', '100'], traffic_fraction=0.4, redis=self.redis)
self.assertEqual(exp._traffic_fraction, 0.40)
def test_is_control(self):
exp = Experiment("trololo", ["yes", "no"], redis=self.redis)
exp.save()
alt = Alternative("yes", exp, redis=self.redis)
self.assertTrue(alt.is_control())
exp.delete()
def test_created_at(self):
exp = Experiment('bench-press', ['joe', 'think'], redis=self.redis)
date = exp.created_at
self.assertIsNone(date)
exp.save()
date = exp.created_at
self.assertTrue(isinstance(date, datetime))
def test_invalid_kpi(self):
ret = Experiment.validate_kpi("!hello-jose")
self.assertFalse(ret)
ret = Experiment.validate_kpi("thunder storm")
self.assertFalse(ret)
ret = Experiment.validate_kpi("&!&&!&")
self.assertFalse(ret)
def test_valid_options(self):
Experiment.find_or_create('red-white', ['red', 'white'],
traffic_fraction=1,
redis=self.redis)
Experiment.find_or_create('red-white-2', ['red', 'white'],
traffic_fraction=0.4,
redis=self.redis)
def test_created_at(self):
exp = Experiment('bench-press', ['joe', 'think'], redis=self.redis)
date = exp.created_at
self.assertIsNone(date)
exp.save()
date = exp.created_at
self.assertTrue(isinstance(date, str))
def test_created_at(self):
exp = Experiment("bench-press", ["joe", "think"], self.redis)
date = exp.created_at()
self.assertIsNone(date)
exp.save()
date = exp.created_at()
self.assertTrue(isinstance(date, datetime))
def test_differing_alternatives_fails(self):
exp = Experiment.find_or_create('load-differing-alts', ['yes', 'zack', 'PBR'], redis=self.redis)
alts = Experiment.load_alternatives(exp.name, redis=self.redis)
self.assertEqual(sorted(alts), sorted(['PBR', 'yes', 'zack']))
with self.assertRaises(ValueError):
exp = Experiment.find_or_create('load-differing-alts', ['kyle', 'zack', 'PBR'], redis=self.redis)
def test_valid_traffic_fractions_save(self):
# test the hidden prop gets set
exp = Experiment.find_or_create('dist-02', ['dist', '100'], traffic_fraction=0.02, redis=self.redis)
self.assertEqual(exp._traffic_fraction, 0.02)
exp = Experiment.find_or_create('dist-100', ['dist', '100'], traffic_fraction=0.4, redis=self.redis)
self.assertEqual(exp._traffic_fraction, 0.40)
def test_participate(self, mock_find_or_create):
exp = Experiment("test", ["no", "yes"], winner=None)
exp.get_alternative = Mock(return_value=Alternative("yes", exp))
mock_find_or_create.return_value = exp
alternative = participate("test", ["no", "yes"], "id1")
self.assertEqual("yes", alternative.name)
self.assertEqual("test", alternative.experiment.name)
def test_participate_with_client_chosen_alternative(self, mock_find_or_create):
exp = Experiment("test", ["no", "yes"], winner=None)
exp.get_alternative = Mock(return_value=Alternative("yes", exp))
mock_find_or_create.return_value = exp
alternative = participate("test", ["no", "yes"], "id1", alternative="yes")
exp.get_alternative.assert_called_once()
self.assertEqual("yes", alternative.name)
def test_convert(self, mock_find):
exp = Experiment("test", ["no", "yes"], winner=None)
exp.convert = Mock(return_value=Alternative("yes", exp))
mock_find.return_value = exp
alternative = convert("test", "id1")
self.assertEqual("yes", alternative.name)
self.assertEqual("test", alternative.experiment.name)
def test_set_winner(self):
exp = Experiment("test-winner", ["1", "2"], self.redis)
exp.set_winner("1")
self.assertTrue(exp.winner is not None)
exp.set_winner("1")
self.assertEqual(exp.winner, "1")
def test_invalid_kpi(self):
ret = Experiment.validate_kpi('!hello-jose')
self.assertFalse(ret)
ret = Experiment.validate_kpi('thunder storm')
self.assertFalse(ret)
ret = Experiment.validate_kpi('&!&&!&')
self.assertFalse(ret)
def test_differing_alternatives_fails(self):
exp = Experiment.find_or_create('load-differing-alts', ['yes', 'zack', 'PBR'], redis=self.redis)
alts = Experiment.load_alternatives(exp.name, redis=self.redis)
self.assertEqual(sorted(alts), sorted(['PBR', 'yes', 'zack']))
with self.assertRaises(ValueError):
exp = Experiment.find_or_create('load-differing-alts', ['kyle', 'zack', 'PBR'], redis=self.redis)
def test_dont_fail_when_participating_in_nondefault_traffic_experiment_without_traffic_param(
self):
Experiment.find_or_create('red-white', ['red', 'white'],
traffic_fraction=0.5,
redis=self.redis)
Experiment.find_or_create('red-white', ['red', 'white'],
redis=self.redis)
def test_differing_alternatives_fails(self):
exp = Experiment.find_or_create("load-differing-alts", ["yes", "zack", "PBR"], self.redis)
alts = Experiment.load_alternatives(exp.name, self.redis)
self.assertEqual(sorted(alts), sorted(["PBR", "yes", "zack"]))
with self.assertRaises(ValueError):
exp = Experiment.find_or_create("load-differing-alts", ["kyle", "zack", "PBR"], self.redis)
def test_set_winner(self):
exp = Experiment('test-winner', ['1', '2'], redis=self.redis)
exp.set_winner('1')
self.assertTrue(exp.winner is not None)
exp.set_winner('1')
self.assertEqual(exp.winner.name, '1')
def test_convert_with_kpi(self, mock_find):
exp = Experiment("test", ["no", "yes"], winner=None)
exp.convert = Mock(return_value=Alternative("yes", exp))
mock_find.return_value = exp
alternative = convert("test", "id1", kpi="goal1")
# TODO: we're not really asserting anything about the KPI
self.assertEqual("yes", alternative.name)
self.assertEqual("test", alternative.experiment.name)
def test_valid_options(self):
opts = {'distribution': 1}
Experiment.find_or_create('red-white', ['red', 'white'], self.redis,
opts)
opts = {'distribution': '1'}
Experiment.find_or_create('red-white', ['red', 'white'], self.redis,
opts)
def test_change_alternatives(self):
exp = Experiment.find_or_create('never-gonna-x', ['let', 'you', 'down'], redis=self.redis)
with self.assertRaises(ValueError):
Experiment.find_or_create('never-gonna-x', ['let', 'you', 'down', 'give', 'you', 'up'], redis=self.redis)
exp.delete()
Experiment.find_or_create('never-gonna-x', ['let', 'you', 'down', 'give', 'you', 'up'], redis=self.redis)
def test_changing_traffic_fraction_succeeds(self):
exp = Experiment.find_or_create('red-white', ['red', 'white'],
traffic_fraction=1,
redis=self.redis)
self.assertEqual(exp._traffic_fraction, 1)
exp = Experiment.find_or_create('red-white', ['red', 'white'],
traffic_fraction=0.4,
redis=self.redis)
self.assertEqual(exp._traffic_fraction, 0.4)
def test_fail_when_changing_traffic(self):
Experiment.find_or_create('red-white', ['red', 'white'],
traffic_fraction=0.8,
redis=self.redis)
with self.assertRaises(ValueError):
Experiment.find_or_create('red-white', ['red', 'white'],
traffic_fraction=0.4,
redis=self.redis)
def test_change_alternatives(self):
exp = Experiment.find_or_create('never-gonna-x', ['let', 'you', 'down'], redis=self.redis)
with self.assertRaises(ValueError):
Experiment.find_or_create('never-gonna-x', ['let', 'you', 'down', 'give', 'you', 'up'], redis=self.redis)
exp.delete()
Experiment.find_or_create('never-gonna-x', ['let', 'you', 'down', 'give', 'you', 'up'], redis=self.redis)
def test_change_alternatives(self):
exp = Experiment.find_or_create("never-gonna-x", ["let", "you", "down"], self.redis)
with self.assertRaises(ValueError):
Experiment.find_or_create("never-gonna-x", ["let", "you", "down", "give", "you", "up"], self.redis)
exp.delete()
Experiment.find_or_create("never-gonna-x", ["let", "you", "down", "give", "you", "up"], self.redis)
def test_valid_traffic_fractions_save(self):
# test the hidden prop gets set
opts = {'traffic_fraction': 0.02}
exp = Experiment.find_or_create('dist-02', ['dist', '100'], self.redis, opts)
self.assertEqual(exp._traffic_fraction, 0.02)
opts = {'traffic_fraction': 0.4}
exp = Experiment.find_or_create('dist-100', ['dist', '100'], self.redis, opts)
self.assertEqual(exp._traffic_fraction, 0.40)
def setUp(self):
self.redis = fakeredis.FakeStrictRedis()
self.alternatives = ["yes", "no"]
self.exp_1 = Experiment("show-something-awesome", self.alternatives, self.redis)
self.exp_2 = Experiment("dales-lagunitas", ["dales", "lagunitas"], self.redis)
self.exp_3 = Experiment("mgd-budheavy", ["mgd", "bud-heavy"], self.redis)
self.exp_1.save()
self.exp_2.save()
self.exp_3.save()
def test_key(self):
key = self.exp_1.key()
self.assertEqual(key, 'sxp:e:show-something-awesome')
key_2 = self.exp_2.key()
self.assertEqual(key_2, 'sxp:e:dales-lagunitas')
exp = Experiment('brews', ['mgd', 'bud-heavy'], redis=self.redis)
key_3 = exp.key()
self.assertEqual(key_3, 'sxp:e:brews')
def setUp(self):
self.redis = fakeredis.FakeStrictRedis()
self.alternatives = ['yes', 'no']
self.exp_1 = Experiment('show-something-awesome', self.alternatives, redis=self.redis)
self.exp_2 = Experiment('dales-lagunitas', ['dales', 'lagunitas'], redis=self.redis)
self.exp_3 = Experiment('mgd-budheavy', ['mgd', 'bud-heavy'], redis=self.redis)
self.exp_1.save()
self.exp_2.save()
self.exp_3.save()
def test_key(self):
key = self.exp_1.key()
self.assertEqual(key, 'sxp:e:show-something-awesome')
key_2 = self.exp_2.key()
self.assertEqual(key_2, 'sxp:e:dales-lagunitas')
exp = Experiment('brews', ['mgd', 'bud-heavy'], redis=self.redis)
key_3 = exp.key()
self.assertEqual(key_3, 'sxp:e:brews')
def _test_initialize_alternatives(self):
# Should throw ValueError
with self.assertRaises(ValueError):
Experiment.initialize_alternatives('n', ['*'], redis=self.redis)
# each item in list should be Alternative Instance
alt_objs = Experiment.initialize_alternatives('n', ['1', '2', '3'])
for alt in alt_objs:
self.assertTrue(isinstance(alt, Alternative))
self.assertTrue(alt.name in ['1', '2', '3'])
def _test_initialize_alternatives(self):
# Should throw ValueError
with self.assertRaises(ValueError):
Experiment.initialize_alternatives('n', ['*'], redis=self.redis)
# each item in list should be Alternative Instance
alt_objs = Experiment.initialize_alternatives('n', ['1', '2', '3'])
for alt in alt_objs:
self.assertTrue(isinstance(alt, Alternative))
self.assertTrue(alt.name in ['1', '2', '3'])
def _test_initialize_alternatives(self):
# Should throw ValueError
with self.assertRaises(ValueError):
Experiment.initialize_alternatives("n", ["*"], self.redis)
# each item in list should be Alternative Instance
alt_objs = Experiment.initialize_alternatives("n", ["1", "2", "3"])
for alt in alt_objs:
self.assertTrue(isinstance(alt, Alternative))
self.assertTrue(alt.name in ["1", "2", "3"])
def test_key(self):
key = self.exp_1.key()
self.assertEqual(key, "sxp:e:show-something-awesome")
key_2 = self.exp_2.key()
self.assertEqual(key_2, "sxp:e:dales-lagunitas")
exp = Experiment("brews", ["mgd", "bud-heavy"], self.redis)
key_3 = exp.key()
self.assertEqual(key_3, "sxp:e:brews")
def test_participate_with_bucket(self, mock_find_or_create, mock_record_participation):
exp = Experiment("test", ["no", "yes"], winner=None)
exp.is_archived = Mock(return_value=False)
exp.existing_alternative = Mock(return_value=False)
exp.is_client_excluded = Mock(return_value=False)
mock_find_or_create.return_value = exp
mock_record_participation.return_value = Alternative("no", exp)
alternative = participate("test", ["no", "yes"], "id1", bucket="no")
self.assertEqual("no", alternative.name)
def test_valid_kpi(self):
ret = Experiment.validate_kpi("hello-jose")
self.assertTrue(ret)
ret = Experiment.validate_kpi("123")
self.assertTrue(ret)
ret = Experiment.validate_kpi("foreigner")
self.assertTrue(ret)
ret = Experiment.validate_kpi("boston")
self.assertTrue(ret)
ret = Experiment.validate_kpi("1_not-two-times-two-times")
self.assertTrue(ret)
def test_valid_kpi(self):
ret = Experiment.validate_kpi('hello-jose')
self.assertTrue(ret)
ret = Experiment.validate_kpi('123')
self.assertTrue(ret)
ret = Experiment.validate_kpi('foreigner')
self.assertTrue(ret)
ret = Experiment.validate_kpi('boston')
self.assertTrue(ret)
ret = Experiment.validate_kpi('1_not-two-times-two-times')
self.assertTrue(ret)
def test_valid_kpi(self):
ret = Experiment.validate_kpi('hello-jose')
self.assertTrue(ret)
ret = Experiment.validate_kpi('123')
self.assertTrue(ret)
ret = Experiment.validate_kpi('foreigner')
self.assertTrue(ret)
ret = Experiment.validate_kpi('boston')
self.assertTrue(ret)
ret = Experiment.validate_kpi('1_not-two-times-two-times')
self.assertTrue(ret)
def test_is_not_valid(self):
not_valid = Experiment.is_valid(1)
self.assertFalse(not_valid)
not_valid = Experiment.is_valid(':123:name')
self.assertFalse(not_valid)
not_valid = Experiment.is_valid('_123name')
self.assertFalse(not_valid)
not_valid = Experiment.is_valid('&123name')
self.assertFalse(not_valid)
def test_is_not_valid(self):
not_valid = Experiment.is_valid(1)
self.assertFalse(not_valid)
not_valid = Experiment.is_valid(':123:name')
self.assertFalse(not_valid)
not_valid = Experiment.is_valid('_123name')
self.assertFalse(not_valid)
not_valid = Experiment.is_valid('&123name')
self.assertFalse(not_valid)
def test_is_not_valid(self):
not_valid = Experiment.is_valid(1)
self.assertFalse(not_valid)
not_valid = Experiment.is_valid(":123:name")
self.assertFalse(not_valid)
not_valid = Experiment.is_valid("_123name")
self.assertFalse(not_valid)
not_valid = Experiment.is_valid("&123name")
self.assertFalse(not_valid)
def test_leaky_delete(self):
exp = Experiment('delete-me-1', self.alternatives, redis=self.redis)
exp.save()
exp2 = Experiment('delete', self.alternatives, redis=self.redis)
exp2.save()
exp2.delete()
exp3 = Experiment.find('delete-me-1', redis=self.redis)
self.assertEqual(exp3.get_alternative_names(), self.alternatives)
def test_invalid_dist(self):
opts = {'distribution': 0}
with self.assertRaises(ValueError):
Experiment.find_or_create('dist-100', ['dist', '100'], self.redis,
opts)
opts = {'distribution': 101}
with self.assertRaises(ValueError):
Experiment.find_or_create('dist-100', ['dist', '100'], self.redis,
opts)
opts = {'distribution': 'x'}
with self.assertRaises(ValueError):
Experiment.find_or_create('dist-100', ['dist', '100'], self.redis,
opts)
# test the hidden prop gets set
opts = {'distribution': 2}
exp = Experiment.find_or_create('dist-20', ['dist', '100'], self.redis,
opts)
self.assertEqual(exp._traffic_dist, .02)
opts = {'distribution': 100}
exp = Experiment.find_or_create('dist-100', ['dist', '100'],
self.redis, opts)
self.assertEqual(exp._traffic_dist, 1.00)
def test_delete(self):
exp = Experiment('delete-me', self.alternatives, redis=self.redis)
exp.save()
exp.delete()
with self.assertRaises(ValueError):
Experiment.find('delete-me', redis=self.redis)
def test_description(self):
exp = Experiment.find_or_create('never-gonna', ['give', 'you', 'up'],
redis=self.redis)
self.assertEqual(exp.description, None)
exp.update_description('hallo')
self.assertEqual(exp.description, 'hallo')
def test_excluded_client(self):
# need proper redis to register the msetbit script
import sixpack.db
sixpack.db.REDIS.flushall()
e = Experiment.find_or_create('excluded-client',
['option-a', 'option-b'],
redis=sixpack.db.REDIS)
self.assertEqual(e.control.participant_count(), 0)
self.assertEqual(e.control.completed_count(), 0)
# force participate 1 proper client on the control alternative
cnil = Client("cnil", redis=sixpack.db.REDIS)
e.control.record_participation(cnil)
e.convert(cnil)
# exclude client, gets control alternative & try to convert
c = Client("c", redis=sixpack.db.REDIS)
e.exclude_client(c)
self.assertTrue(e.control == e.get_alternative(c))
self.assertTrue(None == e.existing_alternative(c))
with self.assertRaises(ValueError):
e.convert(c)
# participation & completed count should be 1
self.assertEqual(e.control.participant_count(), 1)
self.assertEqual(e.control.completed_count(), 1)
def test_set_kpi(self):
exp = Experiment.find_or_create('multi-kpi', ['kpi', '123'], redis=self.redis)
# We shouldn't beable to manually set a KPI. Only via web request
with self.assertRaises(ValueError):
exp.set_kpi('bananza')
# simulate conversion via webrequest
client = Client(100, redis=self.redis)
exp.get_alternative(client)
exp.convert(client, None, 'bananza')
exp2 = Experiment.find_or_create('multi-kpi', ['kpi', '123'], redis=self.redis)
self.assertEqual(exp2.kpi, None)
exp2.set_kpi('bananza')
self.assertEqual(exp2.kpi, 'bananza')
def test_excluded_client(self):
# need proper redis to register the msetbit script
import sixpack.db
sixpack.db.REDIS.flushall()
e = Experiment.find_or_create('excluded-client', ['option-a', 'option-b'], redis=sixpack.db.REDIS)
self.assertEqual(e.control.participant_count(), 0)
self.assertEqual(e.control.completed_count(), 0)
# force participate 1 proper client on the control alternative
cnil = Client("cnil", redis=sixpack.db.REDIS)
e.control.record_participation(cnil)
e.convert(cnil)
# exclude client, gets control alternative & try to convert
c = Client("c", redis=sixpack.db.REDIS)
e.exclude_client(c)
self.assertTrue(e.control == e.get_alternative(c))
self.assertTrue(None == e.existing_alternative(c))
with self.assertRaises(ValueError):
e.convert(c)
# participation & completed count should be 1
self.assertEqual(e.control.participant_count(), 1)
self.assertEqual(e.control.completed_count(), 1)
def test_get_alternative(self):
client = Client(10, redis=self.redis)
exp = Experiment.find_or_create('pause-control', ['w', 'l'],
redis=self.redis)
exp.pause()
# should return control on paused test with no winner
alt = exp.get_alternative(client)
self.assertEqual(alt.name, 'w')
# should return current participation
exp.resume()
# should be one or the other.
# @todo, this test is not really testing anything.
selected_for_client = exp.get_alternative(client)
self.assertIn(selected_for_client.name, ['w', 'l'])
# should check to see if client is participating and only return the same alt
# unsure how to currently test since fakeredis obviously doesn't parse lua
# most likely integration tests
exp.archive()
# test if the redis key is no longer present
self.redis.get(_key('e:{0}:users'.format(exp.name)))
# should return control on archived test
alt = exp.get_alternative(client)
self.assertEqual(alt.name, 'w')
def test_winner(self):
exp = Experiment.find_or_create('test-get-winner', ['1', '2'],
redis=self.redis)
self.assertIsNone(exp.winner)
exp.set_winner('1')
self.assertEqual(exp.winner.name, '1')
def test_reset_winner(self):
exp = Experiment('show-something-reset-winner', self.alternatives, redis=self.redis)
exp.save()
exp.set_winner('yes')
self.assertTrue(exp.winner is not None)
self.assertEqual(exp.winner.name, 'yes')
exp.reset_winner()
self.assertIsNone(exp.winner)
def test_add_kpi(self):
exp = Experiment.find_or_create('multi-kpi-add', ['asdf', '999'], redis=self.redis)
kpi = 'omg-pop'
exp.add_kpi(kpi)
key = "{0}:kpis".format(exp.key(include_kpi=False))
self.assertIn(kpi, self.redis.smembers(key))
exp.delete()
def test_cant_convert_twice(self):
exp = Experiment('test-cant-convert-twice', ['1', '2'], self.app.redis)
client = Client("eric", self.app.redis)
alt = exp.get_alternative(client)
exp.convert(client)
self.assertEqual(exp.total_conversions(), 1)
exp.convert(client, dt=dateutil.parser.parse("2012-01-01"))
self.assertEqual(exp.total_conversions(), 1)
data = exp.objectify_by_period("day")
altdata = [a for a in data["alternatives"]
if a["name"] == alt.name][0]["data"]
total_participants = sum([d["participants"] for d in altdata])
self.assertEqual(total_participants, 1)
total_conversions = sum([d["conversions"] for d in altdata])
self.assertEqual(total_conversions, 1)
def test_find(self):
exp = Experiment('crunches-situps', ['crunches', 'situps'], redis=self.redis)
exp.save()
with self.assertRaises(ValueError):
Experiment.find('this-does-not-exist', redis=self.redis)
try:
Experiment.find('crunches-situps', redis=self.redis)
except:
self.fail('known exp not found')