def testAnonymousUser(self): experiment = Experiment(name="enabled") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() for i in range(100): user = TestUser() in_test = Experiment.test("enabled", user) anonymous_id = user.get_anonymous_id() self.assertNotEquals(None, anonymous_id) in_control = Experiment.control("enabled", user) self.assertEquals(user.get_anonymous_id(), anonymous_id) self.assertNotEquals(in_test, in_control) self.assertEquals(in_test, Experiment.test("enabled", user)) self.assertEquals(user.get_anonymous_id(), anonymous_id) self.assertEquals(in_control, Experiment.control("enabled", user)) self.assertEquals(user.get_anonymous_id(), anonymous_id) if in_test: test_user = user if in_control: control_user = user self.assertTrue(test_user and control_user) user = TestUser() experiment = Experiment(name="disabled") experiment.save() self.assertFalse(Experiment.test("disabled", user)) self.assertEquals(None, user.get_anonymous_id())
def testMissingGoalType(self): anonymous_visitor = AnonymousVisitor() anonymous_visitor.save() goal_type = GoalType(name="existing-goal") goal_type.save() nb_types = GoalType.objects.all().count() nb_records = GoalRecord.objects.all().count() GoalRecord.record('existing-goal', TestUser(anonymous_visitor=anonymous_visitor)) self.assertEquals(nb_records + 1, GoalRecord.objects.all().count()) self.assertEqual(nb_types, GoalType.objects.all().count()) with patch(settings, 'LEAN_AUTOCREATE_GOAL_TYPES', False): GoalRecord.record('inexistant-goal', TestUser(anonymous_visitor=anonymous_visitor)) self.assertEquals(nb_records + 1, GoalRecord.objects.all().count()) self.assertEqual(nb_types, GoalType.objects.all().count()) with patch(settings, 'LEAN_AUTOCREATE_GOAL_TYPES', NotImplemented): GoalRecord.record('inexistant-goal', TestUser(anonymous_visitor=anonymous_visitor)) self.assertEquals(nb_records + 1, GoalRecord.objects.all().count()) self.assertEqual(nb_types, GoalType.objects.all().count()) with patch(settings, 'LEAN_AUTOCREATE_GOAL_TYPES', True): GoalRecord.record('inexistant-goal', TestUser(anonymous_visitor=anonymous_visitor)) self.assertEquals(nb_records + 2, GoalRecord.objects.all().count()) self.assertEqual(nb_types + 1, GoalType.objects.all().count())
def testDisabledExperiment(self): # disabled test (unenrolled user) experiment = Experiment(name="disabled") experiment.save() user = TestUser(username="******") self.assertFalse(Experiment.test("disabled", user)) self.assertTrue(Experiment.control("disabled", user))
def testGoals(self): anonymous_visitor = AnonymousVisitor() anonymous_visitor.save() # required fields for GoalType self.assertRaises(Exception, lambda: GoalType(name=None).save()) goal_type = GoalType(name="test-goal") goal_type.save() # unique constraint on GoalType.name self.assertRaises(Exception, lambda: GoalType(name="test-goal").save()) # required fields for GoalRecord self.assertRaises(Exception, lambda: GoalRecord().save()) self.assertRaises( Exception, lambda: GoalRecord(anonymous_visitor=anonymous_visitor).save()) self.assertRaises(Exception, lambda: GoalRecord(goal_type=goal_type).save()) now = datetime.now() goal_record = GoalRecord(anonymous_visitor=anonymous_visitor, goal_type=goal_type) goal_record.save() self.assertTrue(goal_record.created >= now and goal_record.created <= datetime.now()) # it's OK for the same user to record the same goal multiple times goal_record2 = GoalRecord(anonymous_visitor=anonymous_visitor, goal_type=goal_type) goal_record2.save() nb_records = GoalRecord.objects.all().count() GoalRecord.record('test-goal', TestUser()) self.assertEquals(nb_records, GoalRecord.objects.all().count()) GoalRecord.record('test-goal', TestUser(username='******')) self.assertEquals(nb_records, GoalRecord.objects.all().count()) GoalRecord.record('test-goal', TestUser(anonymous_visitor=anonymous_visitor)) self.assertEquals(nb_records + 1, GoalRecord.objects.all().count()) GoalRecord.record('test-goal', TestUser(anonymous_visitor=anonymous_visitor)) self.assertEquals(nb_records + 2, GoalRecord.objects.all().count())
def testPromotedExperiment(self): # promoted test (unenrolled user) experiment = Experiment(name="promoted") experiment.save() experiment.state = Experiment.PROMOTED_STATE experiment.save() user = TestUser(username="******") self.assertTrue(Experiment.test("promoted", user)) self.assertFalse(Experiment.control("promoted", user))
def testParticipantEnrollment(self): experiment1 = Experiment(name="Experiment 1") experiment1.save() experiment1.state = Experiment.ENABLED_STATE experiment1.save() experiment2 = Experiment(name="Experiment 2") experiment2.save() experiment2.state = Experiment.ENABLED_STATE experiment2.save() num_control1 = 0 num_test1 = 0 num_control2 = 0 num_test2 = 0 for i in range(1000): username = "******" % i in_test1 = Experiment.test(experiment1.name, TestUser(username=username)) self.assertEquals( in_test1, not Experiment.control(experiment1.name, TestUser(username=username))) if in_test1: num_test1 += 1 else: num_control1 += 1 in_test2 = not Experiment.control(experiment2.name, TestUser(username=username)) self.assertEquals( in_test2, Experiment.test(experiment2.name, TestUser(username=username))) if in_test2: num_test2 += 1 else: num_control2 += 1 self.assert_(num_control1 > 400) self.assert_(num_control2 > 400) self.assert_(num_test1 > 400) self.assert_(num_test2 > 400)
def doRenderExperimentToken(self, username, token_tuple, expect_parse_exception=False, expect_render_exception=False): internal_render_result = "rendered" parser = self.mox.CreateMockAnything() child_node_list = self.mox.CreateMockAnything() context = {} user_factory = self.mox.CreateMockAnything() token = self.mox.CreateMockAnything() token.split_contents().AndReturn(token_tuple) parser.parse(('endexperiment', )).MultipleTimes().AndReturn( child_node_list) parser.delete_first_token().MultipleTimes() user_factory.create_user(context).MultipleTimes().AndReturn( TestUser(username=username)) child_node_list.render(context).MultipleTimes().AndReturn( internal_render_result) self.mox.ReplayAll() # HACK: there is no way to make a call optional child_node_list.render(context) parser.parse(('endexperiment', )) parser.delete_first_token() user_factory.create_user(context) do_parse = lambda: experiment(parser, token, user_factory=user_factory) if expect_parse_exception: self.assertRaises(Exception, do_parse) else: node = do_parse() if expect_parse_exception: self.mox.VerifyAll() return None do_render = lambda: node.render(context) if expect_render_exception: self.assertRaises(Exception, do_render) else: actual_render_result = do_render() if expect_render_exception: self.mox.VerifyAll() return None self.mox.VerifyAll() self.assertTrue(actual_render_result == "" or actual_render_result == internal_render_result) in_group = (actual_render_result == internal_render_result) return in_group
def testBotExclusion(self): experiment = Experiment(name="bot_experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() user = TestUser(verified_human=False) participants_count = Participant.objects.all().count() in_test = Experiment.test(experiment.name, user) self.assertEquals(None, user.get_anonymous_id()) self.assertEquals(participants_count, Participant.objects.all().count()) enrollments = user.get_added_enrollments() self.assertEquals(len(enrollments.keys()), 1) self.assertTrue(experiment.name in enrollments.keys())
def testUnknownExperiment(self): # unknown test (unenrolled user) user = TestUser(username="******") # unit tests are always run in DEBUG=False by Django self.assertFalse(Experiment.test("undefined", user)) self.assertTrue(Experiment.control("undefined", user)) try: settings.DEBUG = True self.assertRaises(Exception, lambda: Experiment.test("undefined", user)) self.assertRaises(Exception, lambda: Experiment.control("undefined", user)) finally: settings.DEBUG = False
def doRenderClientSideExperiment(self, context, username, experiment_name): parser = self.mox.CreateMockAnything() user_factory = self.mox.CreateMockAnything() token = self.mox.CreateMockAnything() token.split_contents().AndReturn(("clientsideexperiment", experiment_name)) user_factory.create_user(context).MultipleTimes().AndReturn( TestUser(username=username)) self.mox.ReplayAll() # HACK this is the only way to make a call optional user_factory.create_user(context) node = clientsideexperiment(parser, token, user_factory=user_factory) node.render(context) self.mox.VerifyAll()
def testEnabledPromotedAndDisabledExperiment(self): # enabled test, new user (prove we get both results) experiment = Experiment(name="enabled") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() test_user = None control_user = None for i in range(100): user = TestUser(username="******" % i) in_test = Experiment.test("enabled", user) in_control = Experiment.control("enabled", user) self.assertNotEquals(in_test, in_control) self.assertEquals(in_test, Experiment.test("enabled", user)) self.assertEquals(in_control, Experiment.control("enabled", user)) if in_test: test_user = user if in_control: control_user = user self.assertNotEquals(None, test_user) self.assertNotEquals(None, control_user) self.assertNotEquals(test_user, control_user) # promoted test with an enrolled user # (prove we get both results, then that all users are in test) experiment.state = Experiment.PROMOTED_STATE experiment.save() self.assertFalse(Experiment.control("enabled", control_user)) self.assertFalse(Experiment.control("enabled", test_user)) self.assertTrue(Experiment.test("enabled", control_user)) self.assertTrue(Experiment.test("enabled", test_user)) # disabled test with an enrolled user # (prove we get both results, then that all users are in control) experiment.state = Experiment.DISABLED_STATE experiment.save() self.assertTrue(Experiment.control("enabled", control_user)) self.assertTrue(Experiment.control("enabled", test_user)) self.assertFalse(Experiment.test("enabled", control_user)) self.assertFalse(Experiment.test("enabled", test_user))