def testMissingGoalType(self):
anonymous_visitor = AnonymousVisitor()
anonymous_visitor.save()
goal_type = GoalType(name="existing-goal")
goal_type.save()
nb_types = GoalType.objects.all().count()
nb_records = GoalRecord.objects.all().count()
GoalRecord.record('existing-goal',
TestUser(anonymous_visitor=anonymous_visitor))
self.assertEquals(nb_records + 1, GoalRecord.objects.all().count())
self.assertEqual(nb_types, GoalType.objects.all().count())
with patch(settings, 'LEAN_AUTOCREATE_GOAL_TYPES', False):
GoalRecord.record('inexistant-goal',
TestUser(anonymous_visitor=anonymous_visitor))
self.assertEquals(nb_records + 1, GoalRecord.objects.all().count())
self.assertEqual(nb_types, GoalType.objects.all().count())
with patch(settings, 'LEAN_AUTOCREATE_GOAL_TYPES', NotImplemented):
GoalRecord.record('inexistant-goal',
TestUser(anonymous_visitor=anonymous_visitor))
self.assertEquals(nb_records + 1, GoalRecord.objects.all().count())
self.assertEqual(nb_types, GoalType.objects.all().count())
with patch(settings, 'LEAN_AUTOCREATE_GOAL_TYPES', True):
GoalRecord.record('inexistant-goal',
TestUser(anonymous_visitor=anonymous_visitor))
self.assertEquals(nb_records + 2, GoalRecord.objects.all().count())
self.assertEqual(nb_types + 1, GoalType.objects.all().count())
def testAnonymousUser(self):
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
for i in range(100):
user = TestUser()
in_test = Experiment.test("enabled", user)
anonymous_id = user.get_anonymous_id()
self.assertNotEquals(None, anonymous_id)
in_control = Experiment.control("enabled", user)
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertEquals(in_control, Experiment.control("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
if in_test:
test_user = user
if in_control:
control_user = user
self.assertTrue(test_user and control_user)
user = TestUser()
experiment = Experiment(name="disabled")
experiment.save()
self.assertFalse(Experiment.test("disabled", user))
self.assertEquals(None, user.get_anonymous_id())
def testBotExclusion(self):
experiment = Experiment(name="bot_experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
user = TestUser(verified_human=False)
participants_count = Participant.objects.all().count()
in_test = Experiment.test(experiment.name, user)
self.assertEquals(None, user.get_anonymous_id())
self.assertEquals(participants_count, Participant.objects.all().count())
enrollments = user.get_added_enrollments()
self.assertEquals(len(enrollments.keys()), 1)
self.assertTrue(experiment.name in enrollments.keys())
def testBotExclusion(self):
experiment = Experiment(name="bot_experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
user = TestUser(verified_human=False)
participants_count = Participant.objects.all().count()
in_test = Experiment.test(experiment.name, user)
self.assertEquals(None, user.get_anonymous_id())
self.assertEquals(participants_count,
Participant.objects.all().count())
enrollments = user.get_added_enrollments()
self.assertEquals(len(enrollments.keys()), 1)
self.assertTrue(experiment.name in enrollments.keys())
def testDisabledExperiment(self):
# disabled test (unenrolled user)
experiment = Experiment(name="disabled")
experiment.save()
user = TestUser(username="******")
self.assertFalse(Experiment.test("disabled", user))
self.assertTrue(Experiment.control("disabled", user))
def testGoals(self):
anonymous_visitor = AnonymousVisitor()
anonymous_visitor.save()
# required fields for GoalType
self.assertRaises(Exception, lambda: GoalType(name=None).save())
goal_type = GoalType(name="test-goal")
goal_type.save()
# unique constraint on GoalType.name
self.assertRaises(Exception, lambda: GoalType(name="test-goal").save())
# required fields for GoalRecord
self.assertRaises(Exception, lambda: GoalRecord().save())
self.assertRaises(
Exception,
lambda: GoalRecord(anonymous_visitor=anonymous_visitor).save())
self.assertRaises(Exception,
lambda: GoalRecord(goal_type=goal_type).save())
now = datetime.now()
goal_record = GoalRecord(anonymous_visitor=anonymous_visitor,
goal_type=goal_type)
goal_record.save()
self.assertTrue(goal_record.created >= now
and goal_record.created <= datetime.now())
# it's OK for the same user to record the same goal multiple times
goal_record2 = GoalRecord(anonymous_visitor=anonymous_visitor,
goal_type=goal_type)
goal_record2.save()
nb_records = GoalRecord.objects.all().count()
GoalRecord.record('test-goal', TestUser())
self.assertEquals(nb_records, GoalRecord.objects.all().count())
GoalRecord.record('test-goal', TestUser(username='******'))
self.assertEquals(nb_records, GoalRecord.objects.all().count())
GoalRecord.record('test-goal',
TestUser(anonymous_visitor=anonymous_visitor))
self.assertEquals(nb_records + 1, GoalRecord.objects.all().count())
GoalRecord.record('test-goal',
TestUser(anonymous_visitor=anonymous_visitor))
self.assertEquals(nb_records + 2, GoalRecord.objects.all().count())
def testAnonymousUser(self):
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
for i in range(100):
user = TestUser()
in_test = Experiment.test("enabled", user)
anonymous_id = user.get_anonymous_id()
self.assertNotEqual(None, anonymous_id)
in_control = Experiment.control("enabled", user)
self.assertEqual(user.get_anonymous_id(), anonymous_id)
self.assertNotEqual(in_test, in_control)
self.assertEqual(in_test, Experiment.test("enabled", user))
self.assertEqual(user.get_anonymous_id(), anonymous_id)
self.assertEqual(in_control, Experiment.control("enabled", user))
self.assertEqual(user.get_anonymous_id(), anonymous_id)
if in_test:
test_user = user
if in_control:
control_user = user
self.assertTrue(test_user and control_user)
user = TestUser()
experiment = Experiment(name="disabled")
experiment.save()
self.assertFalse(Experiment.test("disabled", user))
self.assertEqual(None, user.get_anonymous_id())
def testPromotedExperiment(self):
# promoted test (unenrolled user)
experiment = Experiment(name="promoted")
experiment.save()
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
user = TestUser(username="******")
self.assertTrue(Experiment.test("promoted", user))
self.assertFalse(Experiment.control("promoted", user))
def testParticipantEnrollment(self):
experiment1 = Experiment(name="Experiment 1")
experiment1.save()
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
for i in range(1000):
username = "******" % i
in_test1 = Experiment.test(experiment1.name,
TestUser(username=username))
self.assertEquals(
in_test1, not Experiment.control(experiment1.name,
TestUser(username=username)))
if in_test1:
num_test1 += 1
else:
num_control1 += 1
in_test2 = not Experiment.control(experiment2.name,
TestUser(username=username))
self.assertEquals(
in_test2,
Experiment.test(experiment2.name, TestUser(username=username)))
if in_test2:
num_test2 += 1
else:
num_control2 += 1
self.assert_(num_control1 > 400)
self.assert_(num_control2 > 400)
self.assert_(num_test1 > 400)
self.assert_(num_test2 > 400)
def doRenderExperimentToken(self, username, token_tuple,
expect_parse_exception=False,
expect_render_exception=False):
internal_render_result = "rendered"
parser = self.mox.CreateMockAnything()
child_node_list = self.mox.CreateMockAnything()
context = {}
user_factory = self.mox.CreateMockAnything()
token = self.mox.CreateMockAnything()
token.split_contents().AndReturn(token_tuple)
parser.parse(('endexperiment', )).MultipleTimes().AndReturn(
child_node_list)
parser.delete_first_token().MultipleTimes()
user_factory.create_user(context).MultipleTimes().AndReturn(
TestUser(username=username))
child_node_list.render(context).MultipleTimes().AndReturn(
internal_render_result)
self.mox.ReplayAll()
# HACK: there is no way to make a call optional
child_node_list.render(context)
parser.parse(('endexperiment', ))
parser.delete_first_token()
user_factory.create_user(context)
do_parse = lambda: experiment(parser, token, user_factory=user_factory)
if expect_parse_exception:
self.assertRaises(Exception, do_parse)
else:
node = do_parse()
if expect_parse_exception:
self.mox.VerifyAll()
return None
do_render = lambda: node.render(context)
if expect_render_exception:
self.assertRaises(Exception, do_render)
else:
actual_render_result = do_render()
if expect_render_exception:
self.mox.VerifyAll()
return None
self.mox.VerifyAll()
self.assertTrue(actual_render_result == "" or
actual_render_result == internal_render_result)
in_group = (actual_render_result == internal_render_result)
return in_group
def testUnknownExperiment(self):
# unknown test (unenrolled user)
user = TestUser(username="******")
# unit tests are always run in DEBUG=False by Django
self.assertFalse(Experiment.test("undefined", user))
self.assertTrue(Experiment.control("undefined", user))
try:
settings.DEBUG = True
self.assertRaises(Exception,
lambda: Experiment.test("undefined", user))
self.assertRaises(Exception,
lambda: Experiment.control("undefined", user))
finally:
settings.DEBUG = False
def doRenderClientSideExperiment(self, context, username, experiment_name):
parser = self.mox.CreateMockAnything()
user_factory = self.mox.CreateMockAnything()
token = self.mox.CreateMockAnything()
token.split_contents().AndReturn(("clientsideexperiment",
experiment_name))
user_factory.create_user(context).MultipleTimes().AndReturn(
TestUser(username=username))
self.mox.ReplayAll()
# HACK this is the only way to make a call optional
user_factory.create_user(context)
node = clientsideexperiment(parser, token, user_factory=user_factory)
node.render(context)
self.mox.VerifyAll()
def testEnabledPromotedAndDisabledExperiment(self):
# enabled test, new user (prove we get both results)
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
test_user = None
control_user = None
for i in range(100):
user = TestUser(username="******" % i)
in_test = Experiment.test("enabled", user)
in_control = Experiment.control("enabled", user)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(in_control, Experiment.control("enabled", user))
if in_test:
test_user = user
if in_control:
control_user = user
self.assertNotEquals(None, test_user)
self.assertNotEquals(None, control_user)
self.assertNotEquals(test_user, control_user)
# promoted test with an enrolled user
# (prove we get both results, then that all users are in test)
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
self.assertFalse(Experiment.control("enabled", control_user))
self.assertFalse(Experiment.control("enabled", test_user))
self.assertTrue(Experiment.test("enabled", control_user))
self.assertTrue(Experiment.test("enabled", test_user))
# disabled test with an enrolled user
# (prove we get both results, then that all users are in control)
experiment.state = Experiment.DISABLED_STATE
experiment.save()
self.assertTrue(Experiment.control("enabled", control_user))
self.assertTrue(Experiment.control("enabled", test_user))
self.assertFalse(Experiment.test("enabled", control_user))
self.assertFalse(Experiment.test("enabled", test_user))
