def testAnonymousUser(self):
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
for i in range(100):
user = TestUser()
in_test = Experiment.test("enabled", user)
anonymous_id = user.get_anonymous_id()
self.assertNotEqual(None, anonymous_id)
in_control = Experiment.control("enabled", user)
self.assertEqual(user.get_anonymous_id(), anonymous_id)
self.assertNotEqual(in_test, in_control)
self.assertEqual(in_test, Experiment.test("enabled", user))
self.assertEqual(user.get_anonymous_id(), anonymous_id)
self.assertEqual(in_control, Experiment.control("enabled", user))
self.assertEqual(user.get_anonymous_id(), anonymous_id)
if in_test:
test_user = user
if in_control:
control_user = user
self.assertTrue(test_user and control_user)
user = TestUser()
experiment = Experiment(name="disabled")
experiment.save()
self.assertFalse(Experiment.test("disabled", user))
self.assertEqual(None, user.get_anonymous_id())
def testAnonymousUser(self):
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
for i in range(100):
user = TestUser()
in_test = Experiment.test("enabled", user)
anonymous_id = user.get_anonymous_id()
self.assertNotEquals(None, anonymous_id)
in_control = Experiment.control("enabled", user)
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertEquals(in_control, Experiment.control("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
if in_test:
test_user = user
if in_control:
control_user = user
self.assertTrue(test_user and control_user)
user = TestUser()
experiment = Experiment(name="disabled")
experiment.save()
self.assertFalse(Experiment.test("disabled", user))
self.assertEquals(None, user.get_anonymous_id())
def testDisabledExperiment(self):
# disabled test (unenrolled user)
experiment = Experiment(name="disabled")
experiment.save()
user = TestUser(username="******")
self.assertFalse(Experiment.test("disabled", user))
self.assertTrue(Experiment.control("disabled", user))
def testUnknownExperiment(self):
# unknown test (unenrolled user)
user = TestUser(username="******")
# unit tests are always run in DEBUG=False by Django
self.assertFalse(Experiment.test("undefined", user))
self.assertTrue(Experiment.control("undefined", user))
try:
settings.DEBUG = True
self.assertRaises(Exception,
lambda: Experiment.test("undefined", user))
self.assertRaises(Exception,
lambda: Experiment.control("undefined", user))
finally:
settings.DEBUG = False
def testUnknownExperiment(self):
# unknown test (unenrolled user)
user = TestUser(username="******")
# unit tests are always run in DEBUG=False by Django
self.assertFalse(Experiment.test("undefined", user))
self.assertTrue(Experiment.control("undefined", user))
try:
settings.DEBUG = True
self.assertRaises(Exception,
lambda: Experiment.test("undefined", user))
self.assertRaises(Exception,
lambda: Experiment.control("undefined", user))
finally:
settings.DEBUG = False
def testDisabledExperiment(self):
# disabled test (unenrolled user)
experiment = Experiment(name="disabled")
experiment.save()
user = TestUser(username="******")
self.assertFalse(Experiment.test("disabled", user))
self.assertTrue(Experiment.control("disabled", user))
def testPromotedExperiment(self):
# promoted test (unenrolled user)
experiment = Experiment(name="promoted")
experiment.save()
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
user = TestUser(username="******")
self.assertTrue(Experiment.test("promoted", user))
self.assertFalse(Experiment.control("promoted", user))
def testPromotedExperiment(self):
# promoted test (unenrolled user)
experiment = Experiment(name="promoted")
experiment.save()
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
user = TestUser(username="******")
self.assertTrue(Experiment.test("promoted", user))
self.assertFalse(Experiment.control("promoted", user))
def testParticipantEnrollment(self):
experiment1 = Experiment(name="Experiment 1")
experiment1.save()
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
for i in range(1000):
username = "******" % i
in_test1 = Experiment.test(experiment1.name,
TestUser(username=username))
self.assertEquals(
in_test1, not Experiment.control(experiment1.name,
TestUser(username=username)))
if in_test1:
num_test1 += 1
else:
num_control1 += 1
in_test2 = not Experiment.control(experiment2.name,
TestUser(username=username))
self.assertEquals(
in_test2,
Experiment.test(experiment2.name, TestUser(username=username)))
if in_test2:
num_test2 += 1
else:
num_control2 += 1
self.assert_(num_control1 > 400)
self.assert_(num_control2 > 400)
self.assert_(num_test1 > 400)
self.assert_(num_test2 > 400)
def testParticipantEnrollment(self):
experiment1 = Experiment(name="Experiment 1")
experiment1.save()
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
for i in range(1000):
username="******" % i
in_test1 = Experiment.test(experiment1.name, TestUser(username=username))
self.assertEquals(in_test1, not Experiment.control(experiment1.name,
TestUser(username=username)))
if in_test1:
num_test1 += 1
else:
num_control1 += 1
in_test2 = not Experiment.control(experiment2.name, TestUser(username=username))
self.assertEquals(in_test2, Experiment.test(experiment2.name,
TestUser(username=username)))
if in_test2:
num_test2 += 1
else:
num_control2 += 1
self.assert_(num_control1 > 400)
self.assert_(num_control2 > 400)
self.assert_(num_test1 > 400)
self.assert_(num_test2 > 400)
def testBotExclusion(self):
experiment = Experiment(name="bot_experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
user = TestUser(verified_human=False)
participants_count = Participant.objects.all().count()
in_test = Experiment.test(experiment.name, user)
self.assertEquals(None, user.get_anonymous_id())
self.assertEquals(participants_count, Participant.objects.all().count())
enrollments = user.get_added_enrollments()
self.assertEquals(len(enrollments.keys()), 1)
self.assertTrue(experiment.name in enrollments.keys())
def render(self, context):
user = self.get_user(context)
should_render = False
if self.group_name == "test":
should_render = Experiment.test(self.experiment_name, user)
elif self.group_name == "control":
should_render = Experiment.control(self.experiment_name, user)
else:
raise Exception("Unknown Experiment group name : %s" %
self.group_name)
if should_render:
return self.node_list.render(context)
else:
return ""
def render(self, context):
user = self.get_user(context)
should_render = False
if self.group_name == "test":
should_render = Experiment.test(self.experiment_name, user)
elif self.group_name == "control":
should_render = Experiment.control(self.experiment_name, user)
else:
raise Exception("Unknown Experiment group name : %s" %
self.group_name)
if should_render:
return self.node_list.render(context)
else:
return ""
def testBotExclusion(self):
experiment = Experiment(name="bot_experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
user = TestUser(verified_human=False)
participants_count = Participant.objects.all().count()
in_test = Experiment.test(experiment.name, user)
self.assertEquals(None, user.get_anonymous_id())
self.assertEquals(participants_count,
Participant.objects.all().count())
enrollments = user.get_added_enrollments()
self.assertEquals(len(enrollments.keys()), 1)
self.assertTrue(experiment.name in enrollments.keys())
def render(self, context):
"""
Appends to a 'client_side_experiments' variable in the context. It
will be the templates responsibility to render this list into the
Javascript context.
"""
if self.CONTEXT_KEY not in context:
context[self.CONTEXT_KEY] = {}
if self.experiment_name not in context[self.CONTEXT_KEY]:
user = self.create_user(context)
group = None
if Experiment.test(self.experiment_name, user):
group = "test"
elif Experiment.control(self.experiment_name, user):
group = "control"
else:
raise Exception("Unexpected test group for experiment %s" %
self.experiment_name)
context[self.CONTEXT_KEY][self.experiment_name] = group
return ""
def render(self, context):
"""
Appends to a 'client_side_experiments' variable in the context. It
will be the templates responsibility to render this list into the
Javascript context.
"""
if self.CONTEXT_KEY not in context:
context[self.CONTEXT_KEY]= {}
if self.experiment_name not in context[self.CONTEXT_KEY]:
user = self.create_user(context)
group = None
if Experiment.test(self.experiment_name, user):
group = "test"
elif Experiment.control(self.experiment_name, user):
group = "control"
else:
raise Exception("Unexpected test group for experiment %s" %
self.experiment_name)
context[self.CONTEXT_KEY][self.experiment_name] = group
return ""
def testEnabledPromotedAndDisabledExperiment(self):
# enabled test, new user (prove we get both results)
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
test_user = None
control_user = None
for i in range(100):
user = TestUser(username="******" % i)
in_test = Experiment.test("enabled", user)
in_control = Experiment.control("enabled", user)
self.assertNotEqual(in_test, in_control)
self.assertEqual(in_test, Experiment.test("enabled", user))
self.assertEqual(in_control, Experiment.control("enabled", user))
if in_test:
test_user = user
if in_control:
control_user = user
self.assertNotEqual(None, test_user)
self.assertNotEqual(None, control_user)
self.assertNotEqual(test_user, control_user)
# promoted test with an enrolled user
# (prove we get both results, then that all users are in test)
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
self.assertFalse(Experiment.control("enabled", control_user))
self.assertFalse(Experiment.control("enabled", test_user))
self.assertTrue(Experiment.test("enabled", control_user))
self.assertTrue(Experiment.test("enabled", test_user))
# disabled test with an enrolled user
# (prove we get both results, then that all users are in control)
experiment.state = Experiment.DISABLED_STATE
experiment.save()
self.assertTrue(Experiment.control("enabled", control_user))
self.assertTrue(Experiment.control("enabled", test_user))
self.assertFalse(Experiment.test("enabled", control_user))
self.assertFalse(Experiment.test("enabled", test_user))
def testEnabledPromotedAndDisabledExperiment(self):
# enabled test, new user (prove we get both results)
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
test_user = None
control_user = None
for i in range(100):
user = TestUser(username="******" % i)
in_test = Experiment.test("enabled", user)
in_control = Experiment.control("enabled", user)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(in_control, Experiment.control("enabled", user))
if in_test:
test_user = user
if in_control:
control_user = user
self.assertNotEquals(None, test_user)
self.assertNotEquals(None, control_user)
self.assertNotEquals(test_user, control_user)
# promoted test with an enrolled user
# (prove we get both results, then that all users are in test)
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
self.assertFalse(Experiment.control("enabled", control_user))
self.assertFalse(Experiment.control("enabled", test_user))
self.assertTrue(Experiment.test("enabled", control_user))
self.assertTrue(Experiment.test("enabled", test_user))
# disabled test with an enrolled user
# (prove we get both results, then that all users are in control)
experiment.state = Experiment.DISABLED_STATE
experiment.save()
self.assertTrue(Experiment.control("enabled", control_user))
self.assertTrue(Experiment.control("enabled", test_user))
self.assertFalse(Experiment.test("enabled", control_user))
self.assertFalse(Experiment.test("enabled", test_user))
def pirates_v_zombies(request, *args, **kwargs):
if Experiment.test("pirates_v_zombies", WebUser(request)):
view = pirates
else:
view = zombies
return view(request, *args, **kwargs)
