def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.experiment.start_date = (self.experiment.start_date - timedelta(days=5)) self.experiment.save() anonymous_visitor = AnonymousVisitor() anonymous_visitor.save() anonymous_participant = Participant( anonymous_visitor=anonymous_visitor, experiment=self.experiment, group=Participant.TEST_GROUP) anonymous_participant.save() anonymous_participant.enrollment_date = self.experiment.start_date anonymous_participant.save() self.other_experiment = Experiment(name="test_experiment2") self.other_experiment.save() self.other_experiment.state = Experiment.DISABLED_STATE self.other_experiment.save() self.other_experiment.start_date = (date.today() - timedelta(days=7)) self.other_experiment.end_date = (date.today() - timedelta(days=3)) self.other_experiment.save()
def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.experiment.start_date = (self.experiment.start_date - timedelta(days=5)) self.experiment.save() anonymous_visitor = AnonymousVisitor() anonymous_visitor.save() anonymous_participant = Participant(anonymous_visitor=anonymous_visitor, experiment=self.experiment, group=Participant.TEST_GROUP) anonymous_participant.save() anonymous_participant.enrollment_date = self.experiment.start_date anonymous_participant.save() self.other_experiment = Experiment(name="test_experiment2") self.other_experiment.save() self.other_experiment.state = Experiment.DISABLED_STATE self.other_experiment.save() self.other_experiment.start_date = (date.today() - timedelta(days=7)) self.other_experiment.end_date = (date.today() - timedelta(days=3)) self.other_experiment.save()
def testDisabledExperiment(self): # disabled test (unenrolled user) experiment = Experiment(name="disabled") experiment.save() user = TestUser(username="******") self.assertFalse(Experiment.test("disabled", user)) self.assertTrue(Experiment.control("disabled", user))
def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.experiment.start_date = (self.experiment.start_date - timedelta(days=5)) self.experiment.save()
def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.other_experiment = Experiment(name="other_test_experiment") self.other_experiment.save() self.other_experiment.state = Experiment.ENABLED_STATE self.other_experiment.save() self.mox = mox.Mox()
def setUp(self): staff_user = User(username="******", email="*****@*****.**", is_staff=True) staff_user.save() staff_user.set_password("staff") staff_user.save() # self.assertTrue(self.client.login(username='******', # password='******')) self.experiment1 = Experiment(name="experiment 1") self.experiment1.save() self.experiment1.state = Experiment.ENABLED_STATE self.experiment1.save() self.experiment1.start_date -= timedelta(days=5) self.experiment1.save() goal_types = [GoalType.objects.create(name='test_goal_%s' % i) for i in range(3)] for i in range(1, 6): DailyEngagementReport.objects.create(date=days_ago(i), experiment=self.experiment1, control_score=3.2, test_score=2.3, control_group_size=3, test_group_size=5, confidence=93.2) conversion_report = DailyConversionReport.objects.create( date=days_ago(i), experiment=self.experiment1, overall_test_conversion=12, overall_control_conversion=9, test_group_size=39, control_group_size=27, confidence=87.4) for goal_type in goal_types: DailyConversionReportGoalData.objects.create( report=conversion_report, goal_type=goal_type, test_conversion=11, control_conversion=7, confidence=45.3) self.experiment2 = Experiment(name="experiment 2") self.experiment2.save() self.experiment3 = Experiment(name="experiment 3") self.experiment3.save() self.experiment3.state = Experiment.ENABLED_STATE self.experiment3.save() self.experiment3.state = Experiment.PROMOTED_STATE self.experiment3.save()
def testPromotedExperiment(self): # promoted test (unenrolled user) experiment = Experiment(name="promoted") experiment.save() experiment.state = Experiment.PROMOTED_STATE experiment.save() user = TestUser(username="******") self.assertTrue(Experiment.test("promoted", user)) self.assertFalse(Experiment.control("promoted", user))
def render(self, context): user = self.get_user(context) should_render = False if self.group_name == "test": should_render = Experiment.test(self.experiment_name, user) elif self.group_name == "control": should_render = Experiment.control(self.experiment_name, user) else: raise Exception("Unknown Experiment group name : %s" % self.group_name) if should_render: return self.node_list.render(context) else: return ""
def testUnknownExperiment(self): # unknown test (unenrolled user) user = TestUser(username="******") # unit tests are always run in DEBUG=False by Django self.assertFalse(Experiment.test("undefined", user)) self.assertTrue(Experiment.control("undefined", user)) try: settings.DEBUG = True self.assertRaises(Exception, lambda: Experiment.test("undefined", user)) self.assertRaises(Exception, lambda: Experiment.control("undefined", user)) finally: settings.DEBUG = False
class TestManagement(TestCase): def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.experiment.start_date = (self.experiment.start_date - timedelta(days=5)) self.experiment.save() def testManageCommand(self): with patch(settings, 'LEAN_ENGAGEMENT_CALCULATOR', 'django_lean.experiments.testsettings.SimpleEngagementCalculator'): #make sure the manage.py command that generates daily stats work #Running with arguments should raise Exception self.assertRaises(CommandError, update_experiment_reports.Command().handle, "some", "args") #This is what manage.py will call self.runner = update_experiment_reports.Command().run_from_argv #Run the reports self.runner(['manage.py', 'update_experiment_reports']) #Make sure they were generated self.assertEquals(5, DailyEngagementReport.objects.filter( experiment=self.experiment).count()) self.assertEquals(5, DailyConversionReport.objects.filter( experiment=self.experiment).count())
def testBotExclusion(self): experiment = Experiment(name="bot_experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() user = TestUser(verified_human=False) participants_count = Participant.objects.all().count() in_test = Experiment.test(experiment.name, user) self.assertEquals(None, user.get_anonymous_id()) self.assertEquals(participants_count, Participant.objects.all().count()) enrollments = user.get_added_enrollments() self.assertEquals(len(enrollments.keys()), 1) self.assertTrue(experiment.name in enrollments.keys())
def render(self, context): """ Appends to a 'client_side_experiments' variable in the context. It will be the templates responsibility to render this list into the Javascript context. """ if self.CONTEXT_KEY not in context: context[self.CONTEXT_KEY] = {} if self.experiment_name not in context[self.CONTEXT_KEY]: user = self.create_user(context) group = None if Experiment.test(self.experiment_name, user): group = "test" elif Experiment.control(self.experiment_name, user): group = "control" else: raise Exception("Unexpected test group for experiment %s" % self.experiment_name) context[self.CONTEXT_KEY][self.experiment_name] = group return ""
def render(self, context): """ Appends to a 'client_side_experiments' variable in the context. It will be the templates responsibility to render this list into the Javascript context. """ if self.CONTEXT_KEY not in context: context[self.CONTEXT_KEY]= {} if self.experiment_name not in context[self.CONTEXT_KEY]: user = self.create_user(context) group = None if Experiment.test(self.experiment_name, user): group = "test" elif Experiment.control(self.experiment_name, user): group = "control" else: raise Exception("Unexpected test group for experiment %s" % self.experiment_name) context[self.CONTEXT_KEY][self.experiment_name] = group return ""
def testGroupSanity(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) for i in range(100): self.client = Client() response = self.client.get(experiment_url) self.assertEquals(response.status_code, 200) self.assertTrue(response.content.strip().lower() in ("test", "control"))
def setUp(self): staff_user = User(username="******", email="*****@*****.**", is_staff=True) staff_user.save() staff_user.set_password("staff") staff_user.save() # self.assertTrue(self.client.login(username='******', # password='******')) self.experiment1 = Experiment(name="experiment 1") self.experiment1.save() self.experiment1.state = Experiment.ENABLED_STATE self.experiment1.save() self.experiment1.start_date -= timedelta(days=5) self.experiment1.save() goal_types = [ GoalType.objects.create(name='test_goal_%s' % i) for i in range(3) ] for i in range(1, 6): DailyEngagementReport.objects.create(date=days_ago(i), experiment=self.experiment1, control_score=3.2, test_score=2.3, control_group_size=3, test_group_size=5, confidence=93.2) conversion_report = DailyConversionReport.objects.create( date=days_ago(i), experiment=self.experiment1, overall_test_conversion=12, overall_control_conversion=9, test_group_size=39, control_group_size=27, confidence=87.4) for goal_type in goal_types: DailyConversionReportGoalData.objects.create( report=conversion_report, goal_type=goal_type, test_conversion=11, control_conversion=7, confidence=45.3) self.experiment2 = Experiment(name="experiment 2") self.experiment2.save() self.experiment3 = Experiment(name="experiment 3") self.experiment3.save() self.experiment3.state = Experiment.ENABLED_STATE self.experiment3.save() self.experiment3.state = Experiment.PROMOTED_STATE self.experiment3.save()
def testGroupSanity(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) for i in range(100): self.client = Client() response = self.client.get(experiment_url) self.assertEqual(response.status_code, 200) self.assertTrue(response.content.strip().lower() in ("test", "control"))
class TestManagement(TestCase): def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.experiment.start_date = (self.experiment.start_date - timedelta(days=5)) self.experiment.save() def testManageCommand(self): with patch( settings, 'LEAN_ENGAGEMENT_CALCULATOR', 'django_lean.experiments.testsettings.SimpleEngagementCalculator' ): #make sure the manage.py command that generates daily stats work #Running with arguments should raise Exception self.assertRaises(CommandError, update_experiment_reports.Command().handle, "some", "args") #This is what manage.py will call self.runner = update_experiment_reports.Command().run_from_argv #Run the reports self.runner(['manage.py', 'update_experiment_reports']) #Make sure they were generated self.assertEquals( 5, DailyEngagementReport.objects.filter( experiment=self.experiment).count()) self.assertEquals( 5, DailyConversionReport.objects.filter( experiment=self.experiment).count())
def testVerifyHuman(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() other_experiment = Experiment(name="other_experiment") other_experiment.save() other_experiment.state = Experiment.ENABLED_STATE other_experiment.save() self.client = Client() original_participants_count = Participant.objects.all().count() original_anonymous_visitors_count = AnonymousVisitor.objects.all( ).count() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) response = self.client.get(experiment_url) self.assertEquals(response.status_code, 200) self.assertEquals(original_participants_count, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count, AnonymousVisitor.objects.all().count()) confirm_human_url = reverse('experiments.views.confirm_human') response = self.client.get(confirm_human_url) self.assertEquals(response.status_code, 204) self.assertEquals(0, len(response.content)) self.assertEquals(original_participants_count + 1, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) # calling it again does nothing response = self.client.get(confirm_human_url) self.assertEquals(response.status_code, 204) self.assertEquals(0, len(response.content)) self.assertEquals(original_participants_count + 1, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) other_experiment_url = reverse( 'experiments.tests.views.experiment_test', args=[other_experiment.name]) response = self.client.get(other_experiment_url) self.assertEquals(response.status_code, 200) # a new participant is immediately created for the new experiment self.assertEquals(original_participants_count + 2, Participant.objects.all().count()) # the existing anonymous visitor is reused self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) # the new experiment does not cause storage of a temporary enrollment response = self.client.get(confirm_human_url) self.assertEquals(response.status_code, 204) self.assertEquals(0, len(response.content)) self.assertEquals(original_participants_count + 2, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count())
class TestExperimentViews(object): urls = 'experiments.tests.urls' def setUp(self): staff_user = User(username="******", email="*****@*****.**", is_staff=True) staff_user.save() staff_user.set_password("staff") staff_user.save() # self.assertTrue(self.client.login(username='******', # password='******')) self.experiment1 = Experiment(name="experiment 1") self.experiment1.save() self.experiment1.state = Experiment.ENABLED_STATE self.experiment1.save() self.experiment1.start_date -= timedelta(days=5) self.experiment1.save() goal_types = [GoalType.objects.create(name='test_goal_%s' % i) for i in range(3)] for i in range(1, 6): DailyEngagementReport.objects.create(date=days_ago(i), experiment=self.experiment1, control_score=3.2, test_score=2.3, control_group_size=3, test_group_size=5, confidence=93.2) conversion_report = DailyConversionReport.objects.create( date=days_ago(i), experiment=self.experiment1, overall_test_conversion=12, overall_control_conversion=9, test_group_size=39, control_group_size=27, confidence=87.4) for goal_type in goal_types: DailyConversionReportGoalData.objects.create( report=conversion_report, goal_type=goal_type, test_conversion=11, control_conversion=7, confidence=45.3) self.experiment2 = Experiment(name="experiment 2") self.experiment2.save() self.experiment3 = Experiment(name="experiment 3") self.experiment3.save() self.experiment3.state = Experiment.ENABLED_STATE self.experiment3.save() self.experiment3.state = Experiment.PROMOTED_STATE self.experiment3.save() def testListExperimentsView(self): url = reverse('experiments.views.list_experiments') response = self.client.get(url) self.assertEqual(response.status_code, 200) header, data = get_tables(response.content)[0] data.sort() self.assertEqual([u'Name', u'Status', u'Start Date', u'End Date'], header) self.assertEqual([[u'experiment 1', u'Enabled', unicode(self.experiment1.start_date), u'None'], [u'experiment 2', u'Disabled', u'None', u'None'], [u'experiment 3', u'Promoted', unicode(self.experiment3.start_date), unicode(self.experiment3.end_date)]], data) def testAnonymousUserCannotAccessReports(self): self.client.logout() url = reverse('experiments.views.experiment_details', args=['experiment 1']) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) url = reverse('experiments.views.list_experiments') response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) def testRegularUserCannotAccessReports(self): self.client.logout() regular_user = User(username="******", email="*****@*****.**") regular_user.save() regular_user.set_password("joe") regular_user.save() self.assertTrue(self.client.login(username='******', password='******')) url = reverse('experiments.views.experiment_details', args=['experiment 1']) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) url = reverse('experiments.views.list_experiments') response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) def test404IfExperimentDoesntExist(self): url = reverse('experiments.views.experiment_details', args=['inexistant experiment']) response = self.client.get(url) self.assertEqual(response.status_code, 404) def testExperimentDetailsView(self): url = reverse('experiments.views.experiment_details', args=['experiment 1']) response = self.client.get(url) self.assertEqual(response.status_code, 200) tables = get_tables(response.content) experiment_properties = tables.pop(0) self.assertEqual([u'Property', u'Value'], experiment_properties[0]) self.assertEqual([[u'Experiment Name', u'experiment 1'], [u'Status', u'Enabled'], [u'Start Date', unicode(self.experiment1.start_date)], [u'End Date', u'None']], experiment_properties[1]) conversion_summary = tables.pop(0) self.assertEqual([u' ', u'Control Group', u'Test Group', u'Improvement', u'Confidence'], conversion_summary[0]) self.assertEqual([[u'Participants', u'27', u'39', u' ', u' '], [u'test_goal_0', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'], [u'test_goal_1', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'], [u'test_goal_2', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'], [u'Any', u'9 (33.3 %)', u'12 (30.8 %)', u'-7 %', u'87 %']], conversion_summary[1]) engagement_summary = tables.pop(0) self.assertEqual([u' ', u'Control Group', u'Test Group', u'Improvement', u'Confidence'], engagement_summary[0]) self.assertEqual([[u'Participants', u'3', u'5', u' ', u' '], [u'Engagement Score', u'3.2', u'2.3', u'-28 %', u'93 %']], engagement_summary[1]) conversion_details = tables.pop(0) self.assertTrue(conversion_details[0]) self.assertTrue(conversion_details[1]) engagement_details = tables.pop(0) self.assertEqual([u'Report Date', u'Control Group Size', u'Control Group Score', u'Test Group Size', u'Test Group Score', u'Test Group Improvement', u'Confidence'], engagement_details[0]) self.assertEqual([unicode(days_ago(1)), u'3', u'3.2', u'5', u'2.3', u'-28 %', u'93 %'], engagement_details[1][0]) self.assertEqual([], tables) def testVerifyHuman(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() other_experiment = Experiment(name="other_experiment") other_experiment.save() other_experiment.state = Experiment.ENABLED_STATE other_experiment.save() self.client = Client() original_participants_count = Participant.objects.all().count() original_anonymous_visitors_count = AnonymousVisitor.objects.all().count() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) response = self.client.get(experiment_url) self.assertEqual(response.status_code, 200) self.assertEqual(original_participants_count, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count, AnonymousVisitor.objects.all().count()) confirm_human_url = reverse('experiments.views.confirm_human') response = self.client.get(confirm_human_url) self.assertEqual(response.status_code, 204) self.assertEqual(0, len(response.content)) self.assertEqual(original_participants_count+1, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count+1, AnonymousVisitor.objects.all().count()) # calling it again does nothing response = self.client.get(confirm_human_url) self.assertEqual(response.status_code, 204) self.assertEqual(0, len(response.content)) self.assertEqual(original_participants_count+1, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count+1, AnonymousVisitor.objects.all().count()) other_experiment_url = reverse( 'experiments.tests.views.experiment_test', args=[other_experiment.name]) response = self.client.get(other_experiment_url) self.assertEqual(response.status_code, 200) # a new participant is immediately created for the new experiment self.assertEqual(original_participants_count + 2, Participant.objects.all().count()) # the existing anonymous visitor is reused self.assertEqual(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) # the new experiment does not cause storage of a temporary enrollment response = self.client.get(confirm_human_url) self.assertEqual(response.status_code, 204) self.assertEqual(0, len(response.content)) self.assertEqual(original_participants_count+2, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count+1, AnonymousVisitor.objects.all().count()) def testGroupSanity(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) for i in range(100): self.client = Client() response = self.client.get(experiment_url) self.assertEqual(response.status_code, 200) self.assertTrue(response.content.strip().lower() in ("test", "control"))
def testEnabledPromotedAndDisabledExperiment(self): # enabled test, new user (prove we get both results) experiment = Experiment(name="enabled") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() test_user = None control_user = None for i in range(100): user = TestUser(username="******" % i) in_test = Experiment.test("enabled", user) in_control = Experiment.control("enabled", user) self.assertNotEquals(in_test, in_control) self.assertEquals(in_test, Experiment.test("enabled", user)) self.assertEquals(in_control, Experiment.control("enabled", user)) if in_test: test_user = user if in_control: control_user = user self.assertNotEquals(None, test_user) self.assertNotEquals(None, control_user) self.assertNotEquals(test_user, control_user) # promoted test with an enrolled user # (prove we get both results, then that all users are in test) experiment.state = Experiment.PROMOTED_STATE experiment.save() self.assertFalse(Experiment.control("enabled", control_user)) self.assertFalse(Experiment.control("enabled", test_user)) self.assertTrue(Experiment.test("enabled", control_user)) self.assertTrue(Experiment.test("enabled", test_user)) # disabled test with an enrolled user # (prove we get both results, then that all users are in control) experiment.state = Experiment.DISABLED_STATE experiment.save() self.assertTrue(Experiment.control("enabled", control_user)) self.assertTrue(Experiment.control("enabled", test_user)) self.assertFalse(Experiment.test("enabled", control_user)) self.assertFalse(Experiment.test("enabled", test_user))
def testPngResponse(self): experiment = Experiment(name="test-experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() goal_type = GoalType(name='test-goal') goal_type.save() experiment_url = reverse("django_lean.experiments.tests.views.experiment_test", args=[experiment.name]) confirm_human_url = reverse("django_lean.experiments.views.confirm_human") client = Client() # we can call with invalid or inexisting names, the response is the same self.call_goal(client, '') self.call_goal(client, 'unknown-goal') # this is an anonymous visitor not enrolled in an experiment, # so no records should be created self.call_goal(client, goal_type.name) self.assertEqual(0, GoalRecord.objects.filter(goal_type=goal_type).count()) nb_anonymous_visitors = AnonymousVisitor.objects.count() # force the user to be a verified human response = client.get(confirm_human_url) self.assertEqual(response.status_code, 204) # force the anonymous visitor to be enrolled in an experiment response = client.get(experiment_url) self.assertEqual(response.status_code, 200) self.assertEqual(nb_anonymous_visitors + 1, AnonymousVisitor.objects.count()) client.get('/test-experiment/%s' % experiment.name) self.assertEqual(nb_anonymous_visitors + 1, AnonymousVisitor.objects.count()) # now call an existing goal again - it should be recorded self.call_goal(client, goal_type.name) self.assertEqual(1, GoalRecord.objects.filter(goal_type=goal_type).count()) # should be recorded again self.call_goal(client, goal_type.name) self.assertEqual(2, GoalRecord.objects.filter(goal_type=goal_type).count()) # validate that both of the records have the same anonymous_visitor two_goal_records = GoalRecord.objects.filter(goal_type=goal_type) self.assertEqual(two_goal_records[0].anonymous_visitor, two_goal_records[1].anonymous_visitor) # try it with a registered user client = Client() user = User(username="******", email="*****@*****.**") user.set_password("password") user.save() response = client.login(username=user.username, password='******') self.assertTrue(response) # force the registered user to be enrolled in an experiment client.get('/test-experiment/%s' % experiment.name) self.call_goal(client, goal_type.name) # since the user was registered, no new records should be created self.assertEqual(2, GoalRecord.objects.filter(goal_type=goal_type).count())
def testParticipantEnrollment(self): experiment1 = Experiment(name="Experiment 1") experiment1.save() experiment1.state = Experiment.ENABLED_STATE experiment1.save() experiment2 = Experiment(name="Experiment 2") experiment2.save() experiment2.state = Experiment.ENABLED_STATE experiment2.save() num_control1 = 0 num_test1 = 0 num_control2 = 0 num_test2 = 0 for i in range(1000): username="******" % i in_test1 = Experiment.test(experiment1.name, TestUser(username=username)) self.assertEquals(in_test1, not Experiment.control(experiment1.name, TestUser(username=username))) if in_test1: num_test1 += 1 else: num_control1 += 1 in_test2 = not Experiment.control(experiment2.name, TestUser(username=username)) self.assertEquals(in_test2, Experiment.test(experiment2.name, TestUser(username=username))) if in_test2: num_test2 += 1 else: num_control2 += 1 self.assert_(num_control1 > 400) self.assert_(num_control2 > 400) self.assert_(num_test1 > 400) self.assert_(num_test2 > 400)
def testParticipants(self): user1 = User(username="******", email="*****@*****.**") user1.save() user2 = User(username="******", email="*****@*****.**") user2.save() user3 = User(username="******", email="*****@*****.**") user3.save() anonymous_visitor1 = AnonymousVisitor() anonymous_visitor1.save() anonymous_visitor2 = AnonymousVisitor() anonymous_visitor2.save() anonymous_visitor3 = AnonymousVisitor() anonymous_visitor3.save() experiment1 = Experiment(name="Experiment 1") experiment1.save() experiment2 = Experiment(name="Experiment 2") experiment2.save() participant11 = Participant(user=user1, experiment= experiment1, group=Participant.CONTROL_GROUP) participant11.save() participant12 = Participant(user=user1, experiment= experiment2, group=Participant.TEST_GROUP) participant12.save() participant21 = Participant(user=user2, experiment= experiment1, group=Participant.CONTROL_GROUP) participant21.save() participant22 = Participant(user=user2, experiment= experiment2, group=Participant.CONTROL_GROUP) participant22.save() self.assertRaises(Exception, lambda: Participant(user=user2, experiment= experiment2, group=Participant.TEST_GROUP).save()) # create anonymous participants participant31 = Participant(anonymous_visitor=anonymous_visitor1, experiment=experiment1, group=Participant.CONTROL_GROUP) participant31.save() participant32 = Participant(anonymous_visitor=anonymous_visitor1, experiment=experiment2, group=Participant.TEST_GROUP) participant32.save() participant42 = Participant(anonymous_visitor=anonymous_visitor2, experiment=experiment2, group=Participant.CONTROL_GROUP) participant42.save() self.assertRaises(Exception, lambda: Participant(anonymous_visitor=anonymous_visitor1, experiment=experiment1, group=Participant.TEST_GROUP).save()) self.assertRaises(Exception, lambda: Participant(experiment=experiment1, group=Participant.TEST_GROUP).save()) self.assertRaises(Exception, lambda: Participant(user=user3, anonymous_visitor=anonymous_visitor3, experiment=experiment1, group=Participant.TEST_GROUP)) self.assertRaises(Exception, lambda: Participant(user=user3, experiment=experiment1).save()) self.assertNotEquals(participant11.enrollment_date, None) participant11.enrollment_date = None self.assertRaises(Exception, lambda: participant11.save()) participant12.group = None self.assertRaises(Exception, lambda: participant12.save())
def testPngResponse(self): experiment = Experiment(name="test-experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() goal_type = GoalType(name='test-goal') goal_type.save() experiment_url = reverse( "django_lean.experiments.tests.views.experiment_test", args=[experiment.name]) confirm_human_url = reverse( "django_lean.experiments.views.confirm_human") client = Client() # we can call with invalid or inexisting names, the response is the same self.call_goal(client, '') self.call_goal(client, 'unknown-goal') # this is an anonymous visitor not enrolled in an experiment, # so no records should be created self.call_goal(client, goal_type.name) self.assertEquals( 0, GoalRecord.objects.filter(goal_type=goal_type).count()) nb_anonymous_visitors = AnonymousVisitor.objects.count() # force the user to be a verified human response = client.get(confirm_human_url) self.assertEquals(response.status_code, 204) # force the anonymous visitor to be enrolled in an experiment response = client.get(experiment_url) self.assertEquals(response.status_code, 200) self.assertEquals(nb_anonymous_visitors + 1, AnonymousVisitor.objects.count()) client.get('/test-experiment/%s' % experiment.name) self.assertEquals(nb_anonymous_visitors + 1, AnonymousVisitor.objects.count()) # now call an existing goal again - it should be recorded self.call_goal(client, goal_type.name) self.assertEquals( 1, GoalRecord.objects.filter(goal_type=goal_type).count()) # should be recorded again self.call_goal(client, goal_type.name) self.assertEquals( 2, GoalRecord.objects.filter(goal_type=goal_type).count()) # validate that both of the records have the same anonymous_visitor two_goal_records = GoalRecord.objects.filter(goal_type=goal_type) self.assertEquals(two_goal_records[0].anonymous_visitor, two_goal_records[1].anonymous_visitor) # try it with a registered user client = Client() user = User(username="******", email="*****@*****.**") user.set_password("password") user.save() response = client.login(username=user.username, password='******') self.assertTrue(response) # force the registered user to be enrolled in an experiment client.get('/test-experiment/%s' % experiment.name) self.call_goal(client, goal_type.name) # since the user was registered, no new records should be created self.assertEquals( 2, GoalRecord.objects.filter(goal_type=goal_type).count())
def testConversionReportGenerator(self): days = [ datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0) ] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)] # experiment starts on days[2] # other experiment starts on days[0] mocker = mox.Mox() finder = mocker.CreateMockAnything() calculator = mocker.CreateMockAnything() default_data = { Participant.TEST_GROUP: { "count": 110, "conversions": [23, 12, 9] }, Participant.CONTROL_GROUP: { "count": 130, "conversions": [12, 47, 5] } } day_2_data = { Participant.TEST_GROUP: { "count": 12, "conversions": [0, 2, 3] }, Participant.CONTROL_GROUP: { "count": 7, "conversions": [1, 0, 3] } } day_3_data = { Participant.TEST_GROUP: { "count": 5, "conversions": [1, 0, 3] }, Participant.CONTROL_GROUP: { "count": 12, "conversions": [0, 0, 0] } } day_4_data = { Participant.TEST_GROUP: { "count": 0, "conversions": [0, 0, 0] }, Participant.CONTROL_GROUP: { "count": 25, "conversions": [2, 3, 7] } } for day in days[2:7]: data = default_data if day == days[2]: data = day_2_data if day == days[3]: data = day_3_data if day == days[4]: data = day_4_data for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP): mock_participants = mocker.CreateMockAnything() finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants) mock_participants.count().MultipleTimes().AndReturn( data[group]["count"]) for goal_type in goal_types: calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn( data[group]["conversions"][int( goal_type.name)]) calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn( sum(data[group]["conversions"])) mocker.ReplayAll() for d in days[2:7]: ConversionReportGenerator( calculator, finder).generate_daily_report_for_experiment( experiment, d.date()) results = DailyConversionReport.objects.filter( experiment=experiment).order_by('-date') mocker.VerifyAll() self.assertEquals(results.count(), 5) report_days = [d.date() for d in days[2:7]] for i in range(5): self.assertEquals(results[i].date, report_days[4 - i]) # Day 2 self.assertEquals(12, results[4].test_group_size) self.assertEquals(7, results[4].control_group_size) self.assertEquals(5, results[4].overall_test_conversion) self.assertEquals(4, results[4].overall_control_conversion) day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[0])[0].test_conversion self.assertEquals(0, day_2_goal_4_test_conversion) day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[2])[0].control_conversion self.assertEquals(3, day_2_goal_2_control_conversion) # Day 3 self.assertEquals(5, results[3].test_group_size) # Day 4 self.assertEquals(0, results[2].test_group_size) self.assertEquals(None, results[2].confidence) day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter( report=results[2], goal_type=goal_types[0])[0].confidence self.assertEquals(None, day_4_goal_1_confidence) # Day 5 day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter( report=results[1], goal_type=goal_types[0])[0].confidence self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testGetConversionData(self): days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0)] yesterday = date.today() - timedelta(days=1) experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = yesterday experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)] report = DailyConversionReport.objects.create(experiment=experiment, date=yesterday, overall_test_conversion=17, overall_control_conversion=12, test_group_size=139, control_group_size=142, confidence=87.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[0], test_conversion=11, control_conversion=0, confidence=65.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[1], test_conversion=0, control_conversion=21, confidence=None) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[2], test_conversion=23, control_conversion=21, confidence=100) data = get_conversion_data(experiment, yesterday) self.assertEqual(data['date'], yesterday) self.assertTrue("totals" in data) self.assertTrue("goal_types" in data) self.assertEqual(4, len(data["goal_types"].keys())) for goal_type in goal_types[0:3]: self.assertTrue(goal_type.name in data["goal_types"]) goal_type_data = data["goal_types"][goal_type.name] self.assertTrue("test_count" in goal_type_data) self.assertTrue("control_count" in goal_type_data) self.assertTrue("test_rate" in goal_type_data) self.assertTrue("control_rate" in goal_type_data) self.assertTrue("improvement" in goal_type_data) self.assertTrue("confidence" in goal_type_data) self.assertEqual(None, data["goal_types"][goal_types[3].name]) self.assertEqual(139, data["test_group_size"]) self.assertEqual(142, data["control_group_size"]) totals = data["totals"] expected_test_rate = 17. / 139. * 100. expected_control_rate = 12. / 142. * 100. expected_improvement = (expected_test_rate - expected_control_rate) / expected_control_rate * 100. self.assertAlmostEqual(expected_test_rate, totals["test_rate"]) self.assertAlmostEqual(expected_control_rate, totals["control_rate"]) self.assertAlmostEqual(expected_improvement, totals["improvement"]) self.assertAlmostEqual(87.3, totals["confidence"]) self.assertEqual(17, totals["test_count"]) self.assertEqual(12, totals["control_count"]) self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_rate"]) self.assertAlmostEqual(11./139*100., data["goal_types"][goal_types[0].name]["test_rate"]) self.assertEqual(None, data["goal_types"][goal_types[0].name]["improvement"]) self.assertAlmostEqual(65.3, data["goal_types"][goal_types[0].name]["confidence"]) self.assertEqual(11, data["goal_types"][goal_types[0].name]["test_count"]) self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_count"]) self.assertAlmostEqual(21./142*100., data["goal_types"][goal_types[1].name]["control_rate"]) self.assertEqual(None, data["goal_types"][goal_types[1].name]["confidence"]) self.assertEqual(None, data["goal_types"][goal_types[1].name]["improvement"]) self.assertAlmostEqual((23./139-21./142)/(21./142)*100., data["goal_types"][goal_types[2].name]["improvement"])
class TestDailyReports(TestCase): def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.experiment.start_date = (self.experiment.start_date - timedelta(days=5)) self.experiment.save() anonymous_visitor = AnonymousVisitor() anonymous_visitor.save() anonymous_participant = Participant( anonymous_visitor=anonymous_visitor, experiment=self.experiment, group=Participant.TEST_GROUP) anonymous_participant.save() anonymous_participant.enrollment_date = self.experiment.start_date anonymous_participant.save() self.other_experiment = Experiment(name="test_experiment2") self.other_experiment.save() self.other_experiment.state = Experiment.DISABLED_STATE self.other_experiment.save() self.other_experiment.start_date = (date.today() - timedelta(days=7)) self.other_experiment.end_date = (date.today() - timedelta(days=3)) self.other_experiment.save() def testDailyEngagementReport(self): users_test = [] users_control = [] num_control1 = 0 num_test1 = 0 num_control2 = 0 num_test2 = 0 #create users for i in range(5): users_control.append( create_user_in_group(self.experiment, i, Participant.CONTROL_GROUP, date.today() - timedelta(days=i))) users_test.append( create_user_in_group(self.experiment, i, Participant.TEST_GROUP, date.today() - timedelta(days=i))) # users_<test|control>[0] were enrolled today, [1] 1 day ago, etc. report_date = date.today() - timedelta(days=1) expected_engagement_score_calls = { (users_test[1], date.today() - timedelta(days=1), report_date): 3.2, (users_test[2], date.today() - timedelta(days=2), report_date): 2.5, (users_test[3], date.today() - timedelta(days=3), report_date): 4.1, (users_test[4], date.today() - timedelta(days=4), report_date): 0, (users_control[1], date.today() - timedelta(days=1), report_date): 0, (users_control[2], date.today() - timedelta(days=2), report_date): 0, (users_control[3], date.today() - timedelta(days=3), report_date): 0, (users_control[4], date.today() - timedelta(days=4), report_date): 0 } test_case = self class EngagementScoreCalculatorStub(object): def calculate_user_engagement_score(self, user, start_date, end_date): test_case.assertNotEquals(user, None) test_case.assertTrue( expected_engagement_score_calls.has_key( (user, start_date, end_date))) return expected_engagement_score_calls[(user, start_date, end_date)] (EngagementReportGenerator(EngagementScoreCalculatorStub()). generate_daily_report_for_experiment(self.experiment, report_date)) experiment_report = DailyEngagementReport.objects.get( experiment=self.experiment, date=report_date) self.assertAlmostEqual((3.2 + 2.5 + 4.1 + 0) / 4.0, experiment_report.test_score) self.assertAlmostEqual(0.0, experiment_report.control_score) self.assertEquals(4, experiment_report.test_group_size) self.assertEquals(4, experiment_report.control_group_size) self.assertAlmostEqual(96.819293337188498, experiment_report.confidence) def testZeroParticipantExperiment(self): mocker = mox.Mox() engagement_calculator = mocker.CreateMockAnything() mocker.ReplayAll() report_date = date.today() EngagementReportGenerator( engagement_score_calculator=engagement_calculator ).generate_daily_report_for_experiment(self.other_experiment, report_date) experiment_report = DailyEngagementReport.objects.get( experiment=self.other_experiment, date=report_date) mocker.VerifyAll() self.assertEquals(None, experiment_report.test_score) self.assertEquals(None, experiment_report.control_score) self.assertEquals(0, experiment_report.test_group_size) self.assertEquals(0, experiment_report.control_group_size) def testGenerateAllDailyEngagementReports(self): class DummyEngagementCalculator(object): def calculate_user_engagement_score(self, user, start_date, end_date): return 7 engagement_report_generator = EngagementReportGenerator( engagement_score_calculator=DummyEngagementCalculator()) engagement_report_generator.generate_daily_report_for_experiment( self.experiment, date.today() - timedelta(days=2)) engagement_report_generator.generate_daily_report_for_experiment( self.experiment, date.today() - timedelta(days=3)) engagement_report_generator.generate_all_daily_reports() DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=1)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=2)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=3)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=4)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=5)) self.assertEquals( 5, DailyEngagementReport.objects.filter( experiment=self.experiment).count()) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=3)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=4)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=5)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=6)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=7)) self.assertEquals( 5, DailyEngagementReport.objects.filter( experiment=self.other_experiment).count()) def create_goal_record(self, record_datetime, anonymous_visitor, goal_type): record = GoalRecord.objects.create(anonymous_visitor=anonymous_visitor, goal_type=goal_type) record.created = record_datetime record.save() def create_participant(self, anonymous_visitor, experiment, enrollment_date, group): participant = Participant.objects.create( anonymous_visitor=anonymous_visitor, experiment=experiment, group=group) participant.enrollment_date = enrollment_date participant.save() return participant def testParticipantConversionCalculator(self): goal_types = [GoalType.objects.create(name=str(i)) for i in range(3)] anonymous_visitor = AnonymousVisitor.objects.create() participant = self.create_participant( anonymous_visitor=anonymous_visitor, experiment=self.experiment, enrollment_date=self.experiment.start_date + timedelta(days=2), group=Participant.TEST_GROUP) days = [ datetime.combine(self.experiment.start_date + timedelta(days=i), time(hour=12)) for i in range(5) ] nb_goal_records = GoalRecord.objects.all().count() self.create_goal_record(days[0], anonymous_visitor, goal_types[0]) self.create_goal_record(days[0], anonymous_visitor, goal_types[1]) self.create_goal_record(days[1], anonymous_visitor, goal_types[0]) self.create_goal_record(days[1], anonymous_visitor, goal_types[0]) self.create_goal_record(days[2], anonymous_visitor, goal_types[1]) self.create_goal_record(days[3], anonymous_visitor, goal_types[0]) self.create_goal_record(days[4], anonymous_visitor, goal_types[0]) self.create_goal_record(days[4], anonymous_visitor, goal_types[0]) self.assertEquals(nb_goal_records + 8, GoalRecord.objects.all().count()) # wasn't enrolled yet! self.assertEquals( calculate_participant_conversion(participant, goal_types[0], days[0]), 0) self.assertEquals( calculate_participant_conversion(participant, goal_types[1], days[0]), 0) self.assertEquals( calculate_participant_conversion(participant, goal_types[2], days[0]), 0) self.assertEquals( calculate_participant_conversion(participant, None, days[0]), 0) self.assertEquals( calculate_participant_conversion(participant, goal_types[0], days[1]), 0) self.assertEquals( calculate_participant_conversion(participant, goal_types[1], days[1]), 0) self.assertEquals( calculate_participant_conversion(participant, goal_types[2], days[1]), 0) self.assertEquals( calculate_participant_conversion(participant, None, days[1]), 0) # now enrolled self.assertEquals( calculate_participant_conversion(participant, goal_types[0], days[2]), 0) self.assertEquals( calculate_participant_conversion(participant, goal_types[1], days[2]), 1) self.assertEquals( calculate_participant_conversion(participant, goal_types[2], days[2]), 0) # "any" is one self.assertEquals( calculate_participant_conversion(participant, None, days[2]), 1) self.assertEquals( calculate_participant_conversion(participant, goal_types[0], days[3]), 1) self.assertEquals( calculate_participant_conversion(participant, goal_types[1], days[3]), 1) self.assertEquals( calculate_participant_conversion(participant, goal_types[2], days[3]), 0) # "any" is one, even though two different goals were achieved self.assertEquals( calculate_participant_conversion(participant, None, days[3]), 1) # there were three conversions on this day for goal 0, but we only count the first! self.assertEquals( calculate_participant_conversion(participant, goal_types[0], days[4]), 1) self.assertEquals( calculate_participant_conversion(participant, goal_types[1], days[4]), 1) self.assertEquals( calculate_participant_conversion(participant, goal_types[2], days[4]), 0) self.assertEquals( calculate_participant_conversion(participant, None, days[4]), 1) def testGoalTypeConversionCalculator(self): mocker = mox.Mox() participants = [ mocker.CreateMockAnything(), mocker.CreateMockAnything(), mocker.CreateMockAnything() ] goal_type = mocker.CreateMockAnything() report_date = mocker.CreateMockAnything() participant_conversion_calculator = mocker.CreateMockAnything() participant_conversion_calculator( participants[0], goal_type, report_date).InAnyOrder().AndReturn(1) participant_conversion_calculator( participants[1], goal_type, report_date).InAnyOrder().AndReturn(0) participant_conversion_calculator( participants[2], goal_type, report_date).InAnyOrder().AndReturn(1) mocker.ReplayAll() self.assertEquals( 2, calculate_goal_type_conversion(goal_type, participants, report_date, participant_conversion_calculator)) mocker.VerifyAll() def testExperimentGroupParticipantFinder(self): days = [ datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0) ] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() anonymous_visitors = [ AnonymousVisitor.objects.create() for i in range(10) ] experiment_participant_groups = [ [ self.create_participant( anonymous_visitor=anonymous_visitors[0], experiment=experiment, enrollment_date=days[2], group=Participant.TEST_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[1], experiment=experiment, enrollment_date=days[2], group=Participant.CONTROL_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[3], experiment=experiment, enrollment_date=days[3], group=Participant.TEST_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[4], experiment=experiment, enrollment_date=days[4], group=Participant.CONTROL_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[6], experiment=experiment, enrollment_date=days[6], group=Participant.TEST_GROUP) ], [ self.create_participant( anonymous_visitor=anonymous_visitors[0], experiment=other_experiment, enrollment_date=days[0], group=Participant.CONTROL_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[2], experiment=other_experiment, enrollment_date=days[0], group=Participant.TEST_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[5], experiment=other_experiment, enrollment_date=days[4], group=Participant.TEST_GROUP) ] ] ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2]) ex1day2visitors = [p.anonymous_visitor for p in ex1day2] self.assertTrue(anonymous_visitors[0] in ex1day2visitors) self.assertEquals(1, len(ex1day2visitors)) ex1day4test = find_experiment_group_participants( Participant.TEST_GROUP, experiment, days[4]) ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test] self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors) self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors) self.assertEquals(2, len(ex1day4testvisitors)) ex1day4control = find_experiment_group_participants( Participant.CONTROL_GROUP, experiment, days[4]) ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control] self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors) self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors) self.assertEquals(2, len(ex1day4controlvisitors)) ex2day5test = find_experiment_group_participants( Participant.TEST_GROUP, other_experiment, days[5]) ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test] self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors) self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors) self.assertEquals(2, len(ex2day5testvisitors)) def testGetConversionData(self): days = [ datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0) ] yesterday = date.today() - timedelta(days=1) experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = yesterday experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)] report = DailyConversionReport.objects.create( experiment=experiment, date=yesterday, overall_test_conversion=17, overall_control_conversion=12, test_group_size=139, control_group_size=142, confidence=87.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[0], test_conversion=11, control_conversion=0, confidence=65.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[1], test_conversion=0, control_conversion=21, confidence=None) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[2], test_conversion=23, control_conversion=21, confidence=100) data = get_conversion_data(experiment, yesterday) self.assertEquals(data['date'], yesterday) self.assertTrue("totals" in data) self.assertTrue("goal_types" in data) self.assertEquals(4, len(data["goal_types"].keys())) for goal_type in goal_types[0:3]: self.assertTrue(goal_type.name in data["goal_types"]) goal_type_data = data["goal_types"][goal_type.name] self.assertTrue("test_count" in goal_type_data) self.assertTrue("control_count" in goal_type_data) self.assertTrue("test_rate" in goal_type_data) self.assertTrue("control_rate" in goal_type_data) self.assertTrue("improvement" in goal_type_data) self.assertTrue("confidence" in goal_type_data) self.assertEquals(None, data["goal_types"][goal_types[3].name]) self.assertEquals(139, data["test_group_size"]) self.assertEquals(142, data["control_group_size"]) totals = data["totals"] expected_test_rate = 17. / 139. * 100. expected_control_rate = 12. / 142. * 100. expected_improvement = (expected_test_rate - expected_control_rate ) / expected_control_rate * 100. self.assertAlmostEquals(expected_test_rate, totals["test_rate"]) self.assertAlmostEquals(expected_control_rate, totals["control_rate"]) self.assertAlmostEquals(expected_improvement, totals["improvement"]) self.assertAlmostEquals(87.3, totals["confidence"]) self.assertEquals(17, totals["test_count"]) self.assertEquals(12, totals["control_count"]) self.assertEquals( 0, data["goal_types"][goal_types[0].name]["control_rate"]) self.assertAlmostEquals( 11. / 139 * 100., data["goal_types"][goal_types[0].name]["test_rate"]) self.assertEquals( None, data["goal_types"][goal_types[0].name]["improvement"]) self.assertAlmostEquals( 65.3, data["goal_types"][goal_types[0].name]["confidence"]) self.assertEquals(11, data["goal_types"][goal_types[0].name]["test_count"]) self.assertEquals( 0, data["goal_types"][goal_types[0].name]["control_count"]) self.assertAlmostEquals( 21. / 142 * 100., data["goal_types"][goal_types[1].name]["control_rate"]) self.assertEquals(None, data["goal_types"][goal_types[1].name]["confidence"]) self.assertEquals( None, data["goal_types"][goal_types[1].name]["improvement"]) self.assertAlmostEquals( (23. / 139 - 21. / 142) / (21. / 142) * 100., data["goal_types"][goal_types[2].name]["improvement"]) #TODO test with zero participants and check rate == None #TODO sometimes confidence cannot be calculated and must return None. Add a test to verify this. def testConversionReportGenerator(self): days = [ datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0) ] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)] # experiment starts on days[2] # other experiment starts on days[0] mocker = mox.Mox() finder = mocker.CreateMockAnything() calculator = mocker.CreateMockAnything() default_data = { Participant.TEST_GROUP: { "count": 110, "conversions": [23, 12, 9] }, Participant.CONTROL_GROUP: { "count": 130, "conversions": [12, 47, 5] } } day_2_data = { Participant.TEST_GROUP: { "count": 12, "conversions": [0, 2, 3] }, Participant.CONTROL_GROUP: { "count": 7, "conversions": [1, 0, 3] } } day_3_data = { Participant.TEST_GROUP: { "count": 5, "conversions": [1, 0, 3] }, Participant.CONTROL_GROUP: { "count": 12, "conversions": [0, 0, 0] } } day_4_data = { Participant.TEST_GROUP: { "count": 0, "conversions": [0, 0, 0] }, Participant.CONTROL_GROUP: { "count": 25, "conversions": [2, 3, 7] } } for day in days[2:7]: data = default_data if day == days[2]: data = day_2_data if day == days[3]: data = day_3_data if day == days[4]: data = day_4_data for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP): mock_participants = mocker.CreateMockAnything() finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants) mock_participants.count().MultipleTimes().AndReturn( data[group]["count"]) for goal_type in goal_types: calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn( data[group]["conversions"][int( goal_type.name)]) calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn( sum(data[group]["conversions"])) mocker.ReplayAll() for d in days[2:7]: ConversionReportGenerator( calculator, finder).generate_daily_report_for_experiment( experiment, d.date()) results = DailyConversionReport.objects.filter( experiment=experiment).order_by('-date') mocker.VerifyAll() self.assertEquals(results.count(), 5) report_days = [d.date() for d in days[2:7]] for i in range(5): self.assertEquals(results[i].date, report_days[4 - i]) # Day 2 self.assertEquals(12, results[4].test_group_size) self.assertEquals(7, results[4].control_group_size) self.assertEquals(5, results[4].overall_test_conversion) self.assertEquals(4, results[4].overall_control_conversion) day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[0])[0].test_conversion self.assertEquals(0, day_2_goal_4_test_conversion) day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[2])[0].control_conversion self.assertEquals(3, day_2_goal_2_control_conversion) # Day 3 self.assertEquals(5, results[3].test_group_size) # Day 4 self.assertEquals(0, results[2].test_group_size) self.assertEquals(None, results[2].confidence) day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter( report=results[2], goal_type=goal_types[0])[0].confidence self.assertEquals(None, day_4_goal_1_confidence) # Day 5 day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter( report=results[1], goal_type=goal_types[0])[0].confidence self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testExperimentGroupParticipantFinder(self): days = [ datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0) ] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() anonymous_visitors = [ AnonymousVisitor.objects.create() for i in range(10) ] experiment_participant_groups = [ [ self.create_participant( anonymous_visitor=anonymous_visitors[0], experiment=experiment, enrollment_date=days[2], group=Participant.TEST_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[1], experiment=experiment, enrollment_date=days[2], group=Participant.CONTROL_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[3], experiment=experiment, enrollment_date=days[3], group=Participant.TEST_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[4], experiment=experiment, enrollment_date=days[4], group=Participant.CONTROL_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[6], experiment=experiment, enrollment_date=days[6], group=Participant.TEST_GROUP) ], [ self.create_participant( anonymous_visitor=anonymous_visitors[0], experiment=other_experiment, enrollment_date=days[0], group=Participant.CONTROL_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[2], experiment=other_experiment, enrollment_date=days[0], group=Participant.TEST_GROUP), self.create_participant( anonymous_visitor=anonymous_visitors[5], experiment=other_experiment, enrollment_date=days[4], group=Participant.TEST_GROUP) ] ] ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2]) ex1day2visitors = [p.anonymous_visitor for p in ex1day2] self.assertTrue(anonymous_visitors[0] in ex1day2visitors) self.assertEquals(1, len(ex1day2visitors)) ex1day4test = find_experiment_group_participants( Participant.TEST_GROUP, experiment, days[4]) ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test] self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors) self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors) self.assertEquals(2, len(ex1day4testvisitors)) ex1day4control = find_experiment_group_participants( Participant.CONTROL_GROUP, experiment, days[4]) ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control] self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors) self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors) self.assertEquals(2, len(ex1day4controlvisitors)) ex2day5test = find_experiment_group_participants( Participant.TEST_GROUP, other_experiment, days[5]) ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test] self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors) self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors) self.assertEquals(2, len(ex2day5testvisitors))
def testGetConversionData(self): days = [ datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0) ] yesterday = date.today() - timedelta(days=1) experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = yesterday experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)] report = DailyConversionReport.objects.create( experiment=experiment, date=yesterday, overall_test_conversion=17, overall_control_conversion=12, test_group_size=139, control_group_size=142, confidence=87.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[0], test_conversion=11, control_conversion=0, confidence=65.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[1], test_conversion=0, control_conversion=21, confidence=None) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[2], test_conversion=23, control_conversion=21, confidence=100) data = get_conversion_data(experiment, yesterday) self.assertEquals(data['date'], yesterday) self.assertTrue("totals" in data) self.assertTrue("goal_types" in data) self.assertEquals(4, len(data["goal_types"].keys())) for goal_type in goal_types[0:3]: self.assertTrue(goal_type.name in data["goal_types"]) goal_type_data = data["goal_types"][goal_type.name] self.assertTrue("test_count" in goal_type_data) self.assertTrue("control_count" in goal_type_data) self.assertTrue("test_rate" in goal_type_data) self.assertTrue("control_rate" in goal_type_data) self.assertTrue("improvement" in goal_type_data) self.assertTrue("confidence" in goal_type_data) self.assertEquals(None, data["goal_types"][goal_types[3].name]) self.assertEquals(139, data["test_group_size"]) self.assertEquals(142, data["control_group_size"]) totals = data["totals"] expected_test_rate = 17. / 139. * 100. expected_control_rate = 12. / 142. * 100. expected_improvement = (expected_test_rate - expected_control_rate ) / expected_control_rate * 100. self.assertAlmostEquals(expected_test_rate, totals["test_rate"]) self.assertAlmostEquals(expected_control_rate, totals["control_rate"]) self.assertAlmostEquals(expected_improvement, totals["improvement"]) self.assertAlmostEquals(87.3, totals["confidence"]) self.assertEquals(17, totals["test_count"]) self.assertEquals(12, totals["control_count"]) self.assertEquals( 0, data["goal_types"][goal_types[0].name]["control_rate"]) self.assertAlmostEquals( 11. / 139 * 100., data["goal_types"][goal_types[0].name]["test_rate"]) self.assertEquals( None, data["goal_types"][goal_types[0].name]["improvement"]) self.assertAlmostEquals( 65.3, data["goal_types"][goal_types[0].name]["confidence"]) self.assertEquals(11, data["goal_types"][goal_types[0].name]["test_count"]) self.assertEquals( 0, data["goal_types"][goal_types[0].name]["control_count"]) self.assertAlmostEquals( 21. / 142 * 100., data["goal_types"][goal_types[1].name]["control_rate"]) self.assertEquals(None, data["goal_types"][goal_types[1].name]["confidence"]) self.assertEquals( None, data["goal_types"][goal_types[1].name]["improvement"]) self.assertAlmostEquals( (23. / 139 - 21. / 142) / (21. / 142) * 100., data["goal_types"][goal_types[2].name]["improvement"])
class TestExperimentViews(object): urls = 'experiments.tests.urls' def setUp(self): staff_user = User(username="******", email="*****@*****.**", is_staff=True) staff_user.save() staff_user.set_password("staff") staff_user.save() # self.assertTrue(self.client.login(username='******', # password='******')) self.experiment1 = Experiment(name="experiment 1") self.experiment1.save() self.experiment1.state = Experiment.ENABLED_STATE self.experiment1.save() self.experiment1.start_date -= timedelta(days=5) self.experiment1.save() goal_types = [ GoalType.objects.create(name='test_goal_%s' % i) for i in range(3) ] for i in range(1, 6): DailyEngagementReport.objects.create(date=days_ago(i), experiment=self.experiment1, control_score=3.2, test_score=2.3, control_group_size=3, test_group_size=5, confidence=93.2) conversion_report = DailyConversionReport.objects.create( date=days_ago(i), experiment=self.experiment1, overall_test_conversion=12, overall_control_conversion=9, test_group_size=39, control_group_size=27, confidence=87.4) for goal_type in goal_types: DailyConversionReportGoalData.objects.create( report=conversion_report, goal_type=goal_type, test_conversion=11, control_conversion=7, confidence=45.3) self.experiment2 = Experiment(name="experiment 2") self.experiment2.save() self.experiment3 = Experiment(name="experiment 3") self.experiment3.save() self.experiment3.state = Experiment.ENABLED_STATE self.experiment3.save() self.experiment3.state = Experiment.PROMOTED_STATE self.experiment3.save() def testListExperimentsView(self): url = reverse('experiments.views.list_experiments') response = self.client.get(url) self.assertEquals(response.status_code, 200) header, data = get_tables(response.content)[0] data.sort() self.assertEquals([u'Name', u'Status', u'Start Date', u'End Date'], header) self.assertEquals([[ u'experiment 1', u'Enabled', unicode(self.experiment1.start_date), u'None' ], [u'experiment 2', u'Disabled', u'None', u'None'], [ u'experiment 3', u'Promoted', unicode(self.experiment3.start_date), unicode(self.experiment3.end_date) ]], data) def testAnonymousUserCannotAccessReports(self): self.client.logout() url = reverse('experiments.views.experiment_details', args=['experiment 1']) response = self.client.get(url) self.assertEquals(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) url = reverse('experiments.views.list_experiments') response = self.client.get(url) self.assertEquals(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) def testRegularUserCannotAccessReports(self): self.client.logout() regular_user = User(username="******", email="*****@*****.**") regular_user.save() regular_user.set_password("joe") regular_user.save() self.assertTrue( self.client.login(username='******', password='******')) url = reverse('experiments.views.experiment_details', args=['experiment 1']) response = self.client.get(url) self.assertEquals(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) url = reverse('experiments.views.list_experiments') response = self.client.get(url) self.assertEquals(response.status_code, 200) self.assertTrue('<title>Log in' in response.content) def test404IfExperimentDoesntExist(self): url = reverse('experiments.views.experiment_details', args=['inexistant experiment']) response = self.client.get(url) self.assertEquals(response.status_code, 404) def testExperimentDetailsView(self): url = reverse('experiments.views.experiment_details', args=['experiment 1']) response = self.client.get(url) self.assertEquals(response.status_code, 200) tables = get_tables(response.content) experiment_properties = tables.pop(0) self.assertEquals([u'Property', u'Value'], experiment_properties[0]) self.assertEquals( [[u'Experiment Name', u'experiment 1'], [u'Status', u'Enabled'], [u'Start Date', unicode(self.experiment1.start_date)], [u'End Date', u'None']], experiment_properties[1]) conversion_summary = tables.pop(0) self.assertEquals([ u' ', u'Control Group', u'Test Group', u'Improvement', u'Confidence' ], conversion_summary[0]) self.assertEquals( [[u'Participants', u'27', u'39', u' ', u' '], [u'test_goal_0', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'], [u'test_goal_1', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'], [u'test_goal_2', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'], [u'Any', u'9 (33.3 %)', u'12 (30.8 %)', u'-7 %', u'87 %']], conversion_summary[1]) engagement_summary = tables.pop(0) self.assertEquals([ u' ', u'Control Group', u'Test Group', u'Improvement', u'Confidence' ], engagement_summary[0]) self.assertEquals( [[u'Participants', u'3', u'5', u' ', u' '], [u'Engagement Score', u'3.2', u'2.3', u'-28 %', u'93 %']], engagement_summary[1]) conversion_details = tables.pop(0) self.assertTrue(conversion_details[0]) self.assertTrue(conversion_details[1]) engagement_details = tables.pop(0) self.assertEquals([ u'Report Date', u'Control Group Size', u'Control Group Score', u'Test Group Size', u'Test Group Score', u'Test Group Improvement', u'Confidence' ], engagement_details[0]) self.assertEquals([ unicode(days_ago(1)), u'3', u'3.2', u'5', u'2.3', u'-28 %', u'93 %' ], engagement_details[1][0]) self.assertEquals([], tables) def testVerifyHuman(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() other_experiment = Experiment(name="other_experiment") other_experiment.save() other_experiment.state = Experiment.ENABLED_STATE other_experiment.save() self.client = Client() original_participants_count = Participant.objects.all().count() original_anonymous_visitors_count = AnonymousVisitor.objects.all( ).count() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) response = self.client.get(experiment_url) self.assertEquals(response.status_code, 200) self.assertEquals(original_participants_count, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count, AnonymousVisitor.objects.all().count()) confirm_human_url = reverse('experiments.views.confirm_human') response = self.client.get(confirm_human_url) self.assertEquals(response.status_code, 204) self.assertEquals(0, len(response.content)) self.assertEquals(original_participants_count + 1, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) # calling it again does nothing response = self.client.get(confirm_human_url) self.assertEquals(response.status_code, 204) self.assertEquals(0, len(response.content)) self.assertEquals(original_participants_count + 1, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) other_experiment_url = reverse( 'experiments.tests.views.experiment_test', args=[other_experiment.name]) response = self.client.get(other_experiment_url) self.assertEquals(response.status_code, 200) # a new participant is immediately created for the new experiment self.assertEquals(original_participants_count + 2, Participant.objects.all().count()) # the existing anonymous visitor is reused self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) # the new experiment does not cause storage of a temporary enrollment response = self.client.get(confirm_human_url) self.assertEquals(response.status_code, 204) self.assertEquals(0, len(response.content)) self.assertEquals(original_participants_count + 2, Participant.objects.all().count()) self.assertEquals(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) def testGroupSanity(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) for i in range(100): self.client = Client() response = self.client.get(experiment_url) self.assertEquals(response.status_code, 200) self.assertTrue(response.content.strip().lower() in ("test", "control"))
def testParticipants(self): user1 = User(username="******", email="*****@*****.**") user1.save() user2 = User(username="******", email="*****@*****.**") user2.save() user3 = User(username="******", email="*****@*****.**") user3.save() anonymous_visitor1 = AnonymousVisitor() anonymous_visitor1.save() anonymous_visitor2 = AnonymousVisitor() anonymous_visitor2.save() anonymous_visitor3 = AnonymousVisitor() anonymous_visitor3.save() experiment1 = Experiment(name="Experiment 1") experiment1.save() experiment2 = Experiment(name="Experiment 2") experiment2.save() participant11 = Participant(user=user1, experiment=experiment1, group=Participant.CONTROL_GROUP) participant11.save() participant12 = Participant(user=user1, experiment=experiment2, group=Participant.TEST_GROUP) participant12.save() participant21 = Participant(user=user2, experiment=experiment1, group=Participant.CONTROL_GROUP) participant21.save() participant22 = Participant(user=user2, experiment=experiment2, group=Participant.CONTROL_GROUP) participant22.save() self.assertRaises( Exception, lambda: Participant(user=user2, experiment=experiment2, group=Participant.TEST_GROUP).save()) # create anonymous participants participant31 = Participant(anonymous_visitor=anonymous_visitor1, experiment=experiment1, group=Participant.CONTROL_GROUP) participant31.save() participant32 = Participant(anonymous_visitor=anonymous_visitor1, experiment=experiment2, group=Participant.TEST_GROUP) participant32.save() participant42 = Participant(anonymous_visitor=anonymous_visitor2, experiment=experiment2, group=Participant.CONTROL_GROUP) participant42.save() self.assertRaises( Exception, lambda: Participant(anonymous_visitor=anonymous_visitor1, experiment=experiment1, group=Participant.TEST_GROUP).save()) self.assertRaises( Exception, lambda: Participant(experiment=experiment1, group=Participant.TEST_GROUP).save()) self.assertRaises( Exception, lambda: Participant(user=user3, anonymous_visitor=anonymous_visitor3, experiment=experiment1, group=Participant.TEST_GROUP)) self.assertRaises( Exception, lambda: Participant(user=user3, experiment=experiment1).save()) self.assertNotEquals(participant11.enrollment_date, None) participant11.enrollment_date = None self.assertRaises(Exception, lambda: participant11.save()) participant12.group = None self.assertRaises(Exception, lambda: participant12.save())
def testAnonymousUser(self): experiment = Experiment(name="enabled") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() for i in range(100): user = TestUser() in_test = Experiment.test("enabled", user) anonymous_id = user.get_anonymous_id() self.assertNotEquals(None, anonymous_id) in_control = Experiment.control("enabled", user) self.assertEquals(user.get_anonymous_id(), anonymous_id) self.assertNotEquals(in_test, in_control) self.assertEquals(in_test, Experiment.test("enabled", user)) self.assertEquals(user.get_anonymous_id(), anonymous_id) self.assertEquals(in_control, Experiment.control("enabled", user)) self.assertEquals(user.get_anonymous_id(), anonymous_id) if in_test: test_user = user if in_control: control_user = user self.assertTrue(test_user and control_user) user = TestUser() experiment = Experiment(name="disabled") experiment.save() self.assertFalse(Experiment.test("disabled", user)) self.assertEquals(None, user.get_anonymous_id())
def testParticipantEnrollment(self): experiment1 = Experiment(name="Experiment 1") experiment1.save() experiment1.state = Experiment.ENABLED_STATE experiment1.save() experiment2 = Experiment(name="Experiment 2") experiment2.save() experiment2.state = Experiment.ENABLED_STATE experiment2.save() num_control1 = 0 num_test1 = 0 num_control2 = 0 num_test2 = 0 for i in range(1000): username = "******" % i in_test1 = Experiment.test(experiment1.name, TestUser(username=username)) self.assertEquals( in_test1, not Experiment.control(experiment1.name, TestUser(username=username))) if in_test1: num_test1 += 1 else: num_control1 += 1 in_test2 = not Experiment.control(experiment2.name, TestUser(username=username)) self.assertEquals( in_test2, Experiment.test(experiment2.name, TestUser(username=username))) if in_test2: num_test2 += 1 else: num_control2 += 1 self.assert_(num_control1 > 400) self.assert_(num_control2 > 400) self.assert_(num_test1 > 400) self.assert_(num_test2 > 400)
def pirates_v_zombies(request, *args, **kwargs): if Experiment.test("pirates_v_zombies", WebUser(request)): view = pirates else: view = zombies return view(request, *args, **kwargs)
def testExperimentStates(self): experiment1 = Experiment(name="test_experiment_1") experiment1.save() experiment1 = Experiment.objects.get(name="test_experiment_1") self.assertRaises(Exception, lambda: Experiment(name="test_experiment_1").save()) #test default values self.assertEquals(experiment1.state, Experiment.DISABLED_STATE) self.assertEquals(experiment1.start_date, None) self.assertEquals(experiment1.end_date, None) #enable the experiment experiment1.state = Experiment.ENABLED_STATE experiment1.save() self.assertNotEquals(experiment1.start_date, None) self.assertEquals(experiment1.end_date, None) #disable the experiement old_start_date = experiment1.start_date experiment1.state = Experiment.DISABLED_STATE experiment1.save() self.assertEquals(experiment1.start_date, old_start_date) self.assertNotEquals(experiment1.end_date, None) #enable the experiment old_end_date = experiment1.end_date experiment1.state = Experiment.ENABLED_STATE experiment1.save() self.assertEquals(experiment1.start_date, old_start_date) self.assertEquals(experiment1.end_date, old_end_date) #promote the experiment experiment1.state = Experiment.PROMOTED_STATE experiment1.save() self.assertEquals(experiment1.start_date, old_start_date) self.assertEquals(experiment1.end_date, old_end_date) experiment2 = Experiment(name="Experiment 2") experiment2.save() experiment2.state = Experiment.ENABLED_STATE experiment2.save() self.assertEquals(experiment2.start_date, experiment1.start_date) experiment2.state = Experiment.PROMOTED_STATE experiment2.save() self.assertNotEquals(experiment2.start_date, None) self.assertNotEquals(experiment2.end_date, None)
class TestDailyReports(TestCase): def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.experiment.start_date = (self.experiment.start_date - timedelta(days=5)) self.experiment.save() anonymous_visitor = AnonymousVisitor() anonymous_visitor.save() anonymous_participant = Participant(anonymous_visitor=anonymous_visitor, experiment=self.experiment, group=Participant.TEST_GROUP) anonymous_participant.save() anonymous_participant.enrollment_date = self.experiment.start_date anonymous_participant.save() self.other_experiment = Experiment(name="test_experiment2") self.other_experiment.save() self.other_experiment.state = Experiment.DISABLED_STATE self.other_experiment.save() self.other_experiment.start_date = (date.today() - timedelta(days=7)) self.other_experiment.end_date = (date.today() - timedelta(days=3)) self.other_experiment.save() def testDailyEngagementReport(self): users_test = [] users_control = [] num_control1 = 0 num_test1 = 0 num_control2 = 0 num_test2 = 0 #create users for i in range(5): users_control.append(create_user_in_group(self.experiment, i, Participant.CONTROL_GROUP, date.today() - timedelta(days=i))) users_test.append(create_user_in_group(self.experiment, i, Participant.TEST_GROUP, date.today() - timedelta(days=i))) # users_<test|control>[0] were enrolled today, [1] 1 day ago, etc. report_date = date.today() - timedelta(days=1) expected_engagement_score_calls = { (users_test[1], date.today() - timedelta(days=1), report_date): 3.2, (users_test[2], date.today() - timedelta(days=2), report_date): 2.5, (users_test[3], date.today() - timedelta(days=3), report_date): 4.1, (users_test[4], date.today() - timedelta(days=4), report_date): 0, (users_control[1], date.today() - timedelta(days=1), report_date): 0, (users_control[2], date.today() - timedelta(days=2), report_date): 0, (users_control[3], date.today() - timedelta(days=3), report_date): 0, (users_control[4], date.today() - timedelta(days=4), report_date): 0} test_case = self class EngagementScoreCalculatorStub(object): def calculate_user_engagement_score(self, user, start_date, end_date): test_case.assertNotEqual(user, None) test_case.assertTrue((user, start_date, end_date) in expected_engagement_score_calls) return expected_engagement_score_calls[(user, start_date, end_date)] (EngagementReportGenerator(EngagementScoreCalculatorStub()). generate_daily_report_for_experiment(self.experiment, report_date)) experiment_report = DailyEngagementReport.objects.get( experiment=self.experiment, date=report_date) self.assertAlmostEqual((3.2 + 2.5 + 4.1 + 0)/4.0, experiment_report.test_score) self.assertAlmostEqual(0.0, experiment_report.control_score) self.assertEqual(4, experiment_report.test_group_size) self.assertEqual(4, experiment_report.control_group_size) self.assertAlmostEqual(96.819293337188498, experiment_report.confidence) def testZeroParticipantExperiment(self): mocker = mox.Mox() engagement_calculator = mocker.CreateMockAnything() mocker.ReplayAll() report_date = date.today() EngagementReportGenerator(engagement_score_calculator=engagement_calculator).generate_daily_report_for_experiment( self.other_experiment, report_date) experiment_report = DailyEngagementReport.objects.get( experiment=self.other_experiment, date=report_date) mocker.VerifyAll() self.assertEqual(None, experiment_report.test_score) self.assertEqual(None, experiment_report.control_score) self.assertEqual(0, experiment_report.test_group_size) self.assertEqual(0, experiment_report.control_group_size) def testGenerateAllDailyEngagementReports(self): class DummyEngagementCalculator(object): def calculate_user_engagement_score(self, user, start_date, end_date): return 7 engagement_report_generator = EngagementReportGenerator(engagement_score_calculator=DummyEngagementCalculator()) engagement_report_generator.generate_daily_report_for_experiment( self.experiment, date.today() - timedelta(days=2)) engagement_report_generator.generate_daily_report_for_experiment( self.experiment, date.today() - timedelta(days=3)) engagement_report_generator.generate_all_daily_reports() DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=1)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=2)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=3)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=4)) DailyEngagementReport.objects.get(experiment=self.experiment, date=date.today() - timedelta(days=5)) self.assertEqual(5, DailyEngagementReport.objects.filter( experiment=self.experiment).count()) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=3)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=4)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=5)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=6)) DailyEngagementReport.objects.get(experiment=self.other_experiment, date=date.today() - timedelta(days=7)) self.assertEqual(5, DailyEngagementReport.objects.filter( experiment=self.other_experiment).count()) def create_goal_record(self, record_datetime, anonymous_visitor, goal_type): record = GoalRecord.objects.create(anonymous_visitor=anonymous_visitor, goal_type=goal_type) record.created = record_datetime record.save() def create_participant(self, anonymous_visitor, experiment, enrollment_date, group): participant = Participant.objects.create(anonymous_visitor=anonymous_visitor, experiment=experiment, group=group) participant.enrollment_date=enrollment_date participant.save() return participant def testParticipantConversionCalculator(self): goal_types = [GoalType.objects.create(name=str(i)) for i in range(3)] anonymous_visitor = AnonymousVisitor.objects.create() participant = self.create_participant( anonymous_visitor=anonymous_visitor, experiment=self.experiment, enrollment_date=self.experiment.start_date + timedelta(days=2), group=Participant.TEST_GROUP) days = [datetime.combine(self.experiment.start_date + timedelta(days=i), time(hour=12)) for i in range(5)] nb_goal_records = GoalRecord.objects.all().count() self.create_goal_record(days[0], anonymous_visitor, goal_types[0]) self.create_goal_record(days[0], anonymous_visitor, goal_types[1]) self.create_goal_record(days[1], anonymous_visitor, goal_types[0]) self.create_goal_record(days[1], anonymous_visitor, goal_types[0]) self.create_goal_record(days[2], anonymous_visitor, goal_types[1]) self.create_goal_record(days[3], anonymous_visitor, goal_types[0]) self.create_goal_record(days[4], anonymous_visitor, goal_types[0]) self.create_goal_record(days[4], anonymous_visitor, goal_types[0]) self.assertEqual(nb_goal_records + 8, GoalRecord.objects.all().count()) # wasn't enrolled yet! self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[0]), 0) self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[0]), 0) self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[0]), 0) self.assertEqual(calculate_participant_conversion(participant, None, days[0]), 0) self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[1]), 0) self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[1]), 0) self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[1]), 0) self.assertEqual(calculate_participant_conversion(participant, None, days[1]), 0) # now enrolled self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[2]), 0) self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[2]), 1) self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[2]), 0) # "any" is one self.assertEqual(calculate_participant_conversion(participant, None, days[2]), 1) self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[3]), 1) self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[3]), 1) self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[3]), 0) # "any" is one, even though two different goals were achieved self.assertEqual(calculate_participant_conversion(participant, None, days[3]), 1) # there were three conversions on this day for goal 0, but we only count the first! self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[4]), 1) self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[4]), 1) self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[4]), 0) self.assertEqual(calculate_participant_conversion(participant, None, days[4]), 1) def testGoalTypeConversionCalculator(self): mocker = mox.Mox() participants = [mocker.CreateMockAnything(), mocker.CreateMockAnything(), mocker.CreateMockAnything()] goal_type = mocker.CreateMockAnything() report_date = mocker.CreateMockAnything() participant_conversion_calculator = mocker.CreateMockAnything() participant_conversion_calculator( participants[0], goal_type, report_date).InAnyOrder().AndReturn(1) participant_conversion_calculator( participants[1], goal_type, report_date).InAnyOrder().AndReturn(0) participant_conversion_calculator( participants[2], goal_type, report_date).InAnyOrder().AndReturn(1) mocker.ReplayAll() self.assertEqual(2, calculate_goal_type_conversion( goal_type, participants, report_date, participant_conversion_calculator)) mocker.VerifyAll() def testExperimentGroupParticipantFinder(self): days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0)] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)] experiment_participant_groups = [ [ self.create_participant(anonymous_visitor=anonymous_visitors[0], experiment=experiment, enrollment_date=days[2], group=Participant.TEST_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[1], experiment=experiment, enrollment_date=days[2], group=Participant.CONTROL_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[3], experiment=experiment, enrollment_date=days[3], group=Participant.TEST_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[4], experiment=experiment, enrollment_date=days[4], group=Participant.CONTROL_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[6], experiment=experiment, enrollment_date=days[6], group=Participant.TEST_GROUP) ], [ self.create_participant(anonymous_visitor=anonymous_visitors[0], experiment=other_experiment, enrollment_date=days[0], group=Participant.CONTROL_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[2], experiment=other_experiment, enrollment_date=days[0], group=Participant.TEST_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[5], experiment=other_experiment, enrollment_date=days[4], group=Participant.TEST_GROUP) ] ] ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2]) ex1day2visitors = [p.anonymous_visitor for p in ex1day2] self.assertTrue(anonymous_visitors[0] in ex1day2visitors) self.assertEqual(1, len(ex1day2visitors)) ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4]) ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test] self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors) self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors) self.assertEqual(2, len(ex1day4testvisitors)) ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4]) ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control] self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors) self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors) self.assertEqual(2, len(ex1day4controlvisitors)) ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5]) ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test] self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors) self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors) self.assertEqual(2, len(ex2day5testvisitors)) def testGetConversionData(self): days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0)] yesterday = date.today() - timedelta(days=1) experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = yesterday experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)] report = DailyConversionReport.objects.create(experiment=experiment, date=yesterday, overall_test_conversion=17, overall_control_conversion=12, test_group_size=139, control_group_size=142, confidence=87.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[0], test_conversion=11, control_conversion=0, confidence=65.3) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[1], test_conversion=0, control_conversion=21, confidence=None) DailyConversionReportGoalData.objects.create(report=report, goal_type=goal_types[2], test_conversion=23, control_conversion=21, confidence=100) data = get_conversion_data(experiment, yesterday) self.assertEqual(data['date'], yesterday) self.assertTrue("totals" in data) self.assertTrue("goal_types" in data) self.assertEqual(4, len(data["goal_types"].keys())) for goal_type in goal_types[0:3]: self.assertTrue(goal_type.name in data["goal_types"]) goal_type_data = data["goal_types"][goal_type.name] self.assertTrue("test_count" in goal_type_data) self.assertTrue("control_count" in goal_type_data) self.assertTrue("test_rate" in goal_type_data) self.assertTrue("control_rate" in goal_type_data) self.assertTrue("improvement" in goal_type_data) self.assertTrue("confidence" in goal_type_data) self.assertEqual(None, data["goal_types"][goal_types[3].name]) self.assertEqual(139, data["test_group_size"]) self.assertEqual(142, data["control_group_size"]) totals = data["totals"] expected_test_rate = 17. / 139. * 100. expected_control_rate = 12. / 142. * 100. expected_improvement = (expected_test_rate - expected_control_rate) / expected_control_rate * 100. self.assertAlmostEqual(expected_test_rate, totals["test_rate"]) self.assertAlmostEqual(expected_control_rate, totals["control_rate"]) self.assertAlmostEqual(expected_improvement, totals["improvement"]) self.assertAlmostEqual(87.3, totals["confidence"]) self.assertEqual(17, totals["test_count"]) self.assertEqual(12, totals["control_count"]) self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_rate"]) self.assertAlmostEqual(11./139*100., data["goal_types"][goal_types[0].name]["test_rate"]) self.assertEqual(None, data["goal_types"][goal_types[0].name]["improvement"]) self.assertAlmostEqual(65.3, data["goal_types"][goal_types[0].name]["confidence"]) self.assertEqual(11, data["goal_types"][goal_types[0].name]["test_count"]) self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_count"]) self.assertAlmostEqual(21./142*100., data["goal_types"][goal_types[1].name]["control_rate"]) self.assertEqual(None, data["goal_types"][goal_types[1].name]["confidence"]) self.assertEqual(None, data["goal_types"][goal_types[1].name]["improvement"]) self.assertAlmostEqual((23./139-21./142)/(21./142)*100., data["goal_types"][goal_types[2].name]["improvement"]) #TODO test with zero participants and check rate == None #TODO sometimes confidence cannot be calculated and must return None. Add a test to verify this. def testConversionReportGenerator(self): days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0)] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)] # experiment starts on days[2] # other experiment starts on days[0] mocker = mox.Mox() finder = mocker.CreateMockAnything() calculator = mocker.CreateMockAnything() default_data = { Participant.TEST_GROUP: { "count": 110, "conversions": [23, 12, 9] }, Participant.CONTROL_GROUP: { "count": 130, "conversions": [12, 47, 5] } } day_2_data = { Participant.TEST_GROUP: { "count": 12, "conversions": [0, 2, 3] }, Participant.CONTROL_GROUP: { "count": 7, "conversions": [1, 0, 3] } } day_3_data = { Participant.TEST_GROUP: { "count": 5, "conversions": [1, 0, 3] }, Participant.CONTROL_GROUP: { "count": 12, "conversions": [0, 0, 0] } } day_4_data = { Participant.TEST_GROUP: { "count": 0, "conversions": [0, 0, 0] }, Participant.CONTROL_GROUP: { "count": 25, "conversions": [2, 3, 7] } } for day in days[2:7]: data = default_data if day == days[2]: data = day_2_data if day == days[3]: data = day_3_data if day == days[4]: data = day_4_data for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP): mock_participants = mocker.CreateMockAnything() finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants) mock_participants.count().MultipleTimes().AndReturn(data[group]["count"]) for goal_type in goal_types: calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)]) calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"])) mocker.ReplayAll() for d in days[2:7]: ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment( experiment, d.date()) results = DailyConversionReport.objects.filter( experiment=experiment).order_by('-date') mocker.VerifyAll() self.assertEqual(results.count(), 5) report_days = [ d.date() for d in days[2:7]] for i in range(5): self.assertEqual(results[i].date, report_days[4-i]) # Day 2 self.assertEqual(12, results[4].test_group_size) self.assertEqual(7, results[4].control_group_size) self.assertEqual(5, results[4].overall_test_conversion) self.assertEqual(4, results[4].overall_control_conversion) day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[0])[0].test_conversion self.assertEqual(0, day_2_goal_4_test_conversion) day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[2])[0].control_conversion self.assertEqual(3, day_2_goal_2_control_conversion) # Day 3 self.assertEqual(5, results[3].test_group_size) # Day 4 self.assertEqual(0, results[2].test_group_size) self.assertEqual(None, results[2].confidence) day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter( report=results[2], goal_type=goal_types[0])[0].confidence self.assertEqual(None, day_4_goal_1_confidence) # Day 5 day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter( report=results[1], goal_type=goal_types[0])[0].confidence self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testVerifyHuman(self): experiment = Experiment(name="experiment") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() other_experiment = Experiment(name="other_experiment") other_experiment.save() other_experiment.state = Experiment.ENABLED_STATE other_experiment.save() self.client = Client() original_participants_count = Participant.objects.all().count() original_anonymous_visitors_count = AnonymousVisitor.objects.all().count() experiment_url = reverse('experiments.tests.views.experiment_test', args=[experiment.name]) response = self.client.get(experiment_url) self.assertEqual(response.status_code, 200) self.assertEqual(original_participants_count, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count, AnonymousVisitor.objects.all().count()) confirm_human_url = reverse('experiments.views.confirm_human') response = self.client.get(confirm_human_url) self.assertEqual(response.status_code, 204) self.assertEqual(0, len(response.content)) self.assertEqual(original_participants_count+1, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count+1, AnonymousVisitor.objects.all().count()) # calling it again does nothing response = self.client.get(confirm_human_url) self.assertEqual(response.status_code, 204) self.assertEqual(0, len(response.content)) self.assertEqual(original_participants_count+1, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count+1, AnonymousVisitor.objects.all().count()) other_experiment_url = reverse( 'experiments.tests.views.experiment_test', args=[other_experiment.name]) response = self.client.get(other_experiment_url) self.assertEqual(response.status_code, 200) # a new participant is immediately created for the new experiment self.assertEqual(original_participants_count + 2, Participant.objects.all().count()) # the existing anonymous visitor is reused self.assertEqual(original_anonymous_visitors_count + 1, AnonymousVisitor.objects.all().count()) # the new experiment does not cause storage of a temporary enrollment response = self.client.get(confirm_human_url) self.assertEqual(response.status_code, 204) self.assertEqual(0, len(response.content)) self.assertEqual(original_participants_count+2, Participant.objects.all().count()) self.assertEqual(original_anonymous_visitors_count+1, AnonymousVisitor.objects.all().count())
def testExperimentGroupParticipantFinder(self): days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0)] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)] experiment_participant_groups = [ [ self.create_participant(anonymous_visitor=anonymous_visitors[0], experiment=experiment, enrollment_date=days[2], group=Participant.TEST_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[1], experiment=experiment, enrollment_date=days[2], group=Participant.CONTROL_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[3], experiment=experiment, enrollment_date=days[3], group=Participant.TEST_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[4], experiment=experiment, enrollment_date=days[4], group=Participant.CONTROL_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[6], experiment=experiment, enrollment_date=days[6], group=Participant.TEST_GROUP) ], [ self.create_participant(anonymous_visitor=anonymous_visitors[0], experiment=other_experiment, enrollment_date=days[0], group=Participant.CONTROL_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[2], experiment=other_experiment, enrollment_date=days[0], group=Participant.TEST_GROUP), self.create_participant(anonymous_visitor=anonymous_visitors[5], experiment=other_experiment, enrollment_date=days[4], group=Participant.TEST_GROUP) ] ] ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2]) ex1day2visitors = [p.anonymous_visitor for p in ex1day2] self.assertTrue(anonymous_visitors[0] in ex1day2visitors) self.assertEqual(1, len(ex1day2visitors)) ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4]) ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test] self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors) self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors) self.assertEqual(2, len(ex1day4testvisitors)) ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4]) ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control] self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors) self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors) self.assertEqual(2, len(ex1day4controlvisitors)) ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5]) ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test] self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors) self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors) self.assertEqual(2, len(ex2day5testvisitors))
def testExperimentStates(self): experiment1 = Experiment(name="test_experiment_1") experiment1.save() experiment1 = Experiment.objects.get(name="test_experiment_1") self.assertRaises(Exception, lambda: Experiment(name="test_experiment_1").save()) #test default values self.assertEquals(experiment1.state, Experiment.DISABLED_STATE) self.assertEquals(experiment1.start_date, None) self.assertEquals(experiment1.end_date, None) #enable the experiment experiment1.state = Experiment.ENABLED_STATE experiment1.save() self.assertNotEquals(experiment1.start_date, None) self.assertEquals(experiment1.end_date, None) #disable the experiement old_start_date = experiment1.start_date experiment1.state = Experiment.DISABLED_STATE experiment1.save() self.assertEquals(experiment1.start_date, old_start_date) self.assertNotEquals(experiment1.end_date, None) #enable the experiment old_end_date = experiment1.end_date experiment1.state = Experiment.ENABLED_STATE experiment1.save() self.assertEquals(experiment1.start_date, old_start_date) self.assertEquals(experiment1.end_date, old_end_date) #promote the experiment experiment1.state = Experiment.PROMOTED_STATE experiment1.save() self.assertEquals(experiment1.start_date, old_start_date) self.assertEquals(experiment1.end_date, old_end_date) experiment2 = Experiment(name= "Experiment 2") experiment2.save() experiment2.state = Experiment.ENABLED_STATE experiment2.save() self.assertEquals(experiment2.start_date, experiment1.start_date) experiment2.state = Experiment.PROMOTED_STATE experiment2.save() self.assertNotEquals(experiment2.start_date, None) self.assertNotEquals(experiment2.end_date, None)
def testConversionReportGenerator(self): days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12)) for i in range(-7, 0)] experiment = Experiment(name="experiment1") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() experiment.start_date = days[2].date() experiment.save() other_experiment = Experiment(name="experiment2") other_experiment.save() other_experiment.state = Experiment.DISABLED_STATE other_experiment.save() other_experiment.start_date = days[0].date() other_experiment.end_date = days[4].date() other_experiment.save() goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)] # experiment starts on days[2] # other experiment starts on days[0] mocker = mox.Mox() finder = mocker.CreateMockAnything() calculator = mocker.CreateMockAnything() default_data = { Participant.TEST_GROUP: { "count": 110, "conversions": [23, 12, 9] }, Participant.CONTROL_GROUP: { "count": 130, "conversions": [12, 47, 5] } } day_2_data = { Participant.TEST_GROUP: { "count": 12, "conversions": [0, 2, 3] }, Participant.CONTROL_GROUP: { "count": 7, "conversions": [1, 0, 3] } } day_3_data = { Participant.TEST_GROUP: { "count": 5, "conversions": [1, 0, 3] }, Participant.CONTROL_GROUP: { "count": 12, "conversions": [0, 0, 0] } } day_4_data = { Participant.TEST_GROUP: { "count": 0, "conversions": [0, 0, 0] }, Participant.CONTROL_GROUP: { "count": 25, "conversions": [2, 3, 7] } } for day in days[2:7]: data = default_data if day == days[2]: data = day_2_data if day == days[3]: data = day_3_data if day == days[4]: data = day_4_data for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP): mock_participants = mocker.CreateMockAnything() finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants) mock_participants.count().MultipleTimes().AndReturn(data[group]["count"]) for goal_type in goal_types: calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)]) calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"])) mocker.ReplayAll() for d in days[2:7]: ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment( experiment, d.date()) results = DailyConversionReport.objects.filter( experiment=experiment).order_by('-date') mocker.VerifyAll() self.assertEqual(results.count(), 5) report_days = [ d.date() for d in days[2:7]] for i in range(5): self.assertEqual(results[i].date, report_days[4-i]) # Day 2 self.assertEqual(12, results[4].test_group_size) self.assertEqual(7, results[4].control_group_size) self.assertEqual(5, results[4].overall_test_conversion) self.assertEqual(4, results[4].overall_control_conversion) day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[0])[0].test_conversion self.assertEqual(0, day_2_goal_4_test_conversion) day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter( report=results[4], goal_type=goal_types[2])[0].control_conversion self.assertEqual(3, day_2_goal_2_control_conversion) # Day 3 self.assertEqual(5, results[3].test_group_size) # Day 4 self.assertEqual(0, results[2].test_group_size) self.assertEqual(None, results[2].confidence) day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter( report=results[2], goal_type=goal_types[0])[0].confidence self.assertEqual(None, day_4_goal_1_confidence) # Day 5 day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter( report=results[1], goal_type=goal_types[0])[0].confidence self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testAnonymousUser(self): experiment = Experiment(name="enabled") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() for i in range(100): user = TestUser() in_test = Experiment.test("enabled", user) anonymous_id = user.get_anonymous_id() self.assertNotEqual(None, anonymous_id) in_control = Experiment.control("enabled", user) self.assertEqual(user.get_anonymous_id(), anonymous_id) self.assertNotEqual(in_test, in_control) self.assertEqual(in_test, Experiment.test("enabled", user)) self.assertEqual(user.get_anonymous_id(), anonymous_id) self.assertEqual(in_control, Experiment.control("enabled", user)) self.assertEqual(user.get_anonymous_id(), anonymous_id) if in_test: test_user = user if in_control: control_user = user self.assertTrue(test_user and control_user) user = TestUser() experiment = Experiment(name="disabled") experiment.save() self.assertFalse(Experiment.test("disabled", user)) self.assertEqual(None, user.get_anonymous_id())
def testEnabledPromotedAndDisabledExperiment(self): # enabled test, new user (prove we get both results) experiment = Experiment(name="enabled") experiment.save() experiment.state = Experiment.ENABLED_STATE experiment.save() test_user = None control_user = None for i in range(100): user = TestUser(username="******" % i) in_test = Experiment.test("enabled", user) in_control = Experiment.control("enabled", user) self.assertNotEqual(in_test, in_control) self.assertEqual(in_test, Experiment.test("enabled", user)) self.assertEqual(in_control, Experiment.control("enabled", user)) if in_test: test_user = user if in_control: control_user = user self.assertNotEqual(None, test_user) self.assertNotEqual(None, control_user) self.assertNotEqual(test_user, control_user) # promoted test with an enrolled user # (prove we get both results, then that all users are in test) experiment.state = Experiment.PROMOTED_STATE experiment.save() self.assertFalse(Experiment.control("enabled", control_user)) self.assertFalse(Experiment.control("enabled", test_user)) self.assertTrue(Experiment.test("enabled", control_user)) self.assertTrue(Experiment.test("enabled", test_user)) # disabled test with an enrolled user # (prove we get both results, then that all users are in control) experiment.state = Experiment.DISABLED_STATE experiment.save() self.assertTrue(Experiment.control("enabled", control_user)) self.assertTrue(Experiment.control("enabled", test_user)) self.assertFalse(Experiment.test("enabled", control_user)) self.assertFalse(Experiment.test("enabled", test_user))
class ExperimentTagsTest(TestCase): urls = 'django_lean.experiments.tests.urls' def setUp(self): self.experiment = Experiment(name="test_experiment") self.experiment.save() self.experiment.state = Experiment.ENABLED_STATE self.experiment.save() self.other_experiment = Experiment(name="other_test_experiment") self.other_experiment.save() self.other_experiment.state = Experiment.ENABLED_STATE self.other_experiment.save() self.mox = mox.Mox() def testIntegrationWithAnonymousVisitor(self): self.doTestIntegration( url=reverse('django_lean.experiments.tests.views.experiment_test', args=[self.experiment.name]), client_factory=lambda i: Client()) self.doTestIntegration( url=reverse('django_lean.experiments.tests.views.clientsideexperiment_test', args=[self.other_experiment.name]), client_factory=lambda i: Client()) def testIntegrationWithRegisteredUser(self): def create_registered_user_client(i): user = User(username="******" % i, email="*****@*****.**" % i) user.set_password("password") user.save() client = Client() if not client.login(username=user.username, password="******"): raise Exception("login failure") return client self.doTestIntegration( url=reverse('django_lean.experiments.tests.views.experiment_test', args=[self.experiment.name]), client_factory=create_registered_user_client) # we wrap our factory to ensure that the users are created # with new names self.doTestIntegration( url=reverse('django_lean.experiments.tests.views.clientsideexperiment_test', args=[self.other_experiment.name]), client_factory=lambda i: create_registered_user_client(1000 + i)) def doTestIntegration(self, url, client_factory): confirm_human_url = reverse('django_lean.experiments.views.confirm_human') found_control = False found_test = False for i in range(100): client = client_factory(i) response = client.get(confirm_human_url) self.assertEquals(204, response.status_code) response = client.get(url) self.assertEquals(200, response.status_code) in_test = "test" in response.content.lower() in_control = "control" in response.content.lower() self.assertTrue(in_test != in_control) found_control = found_control or in_control found_test = found_test or in_test self.assertTrue(found_control) self.assertTrue(found_test) def testIllegalGroupName(self): self.doRenderExperiment("username", "foo group", expect_render_exception=True) def testBadSyntax(self): self.doRenderExperimentToken( "username", ("experiment", self.experiment.name, "group", "yeah"), expect_parse_exception=True) def doRenderExperimentToken(self, username, token_tuple, expect_parse_exception=False, expect_render_exception=False): internal_render_result = "rendered" parser = self.mox.CreateMockAnything() child_node_list = self.mox.CreateMockAnything() context = {} user_factory = self.mox.CreateMockAnything() token = self.mox.CreateMockAnything() token.split_contents().AndReturn(token_tuple) parser.parse(('endexperiment', )).MultipleTimes().AndReturn( child_node_list) parser.delete_first_token().MultipleTimes() user_factory.create_user(context).MultipleTimes().AndReturn( TestUser(username=username)) child_node_list.render(context).MultipleTimes().AndReturn( internal_render_result) self.mox.ReplayAll() # HACK: there is no way to make a call optional child_node_list.render(context) parser.parse(('endexperiment', )) parser.delete_first_token() user_factory.create_user(context) do_parse = lambda: experiment(parser, token, user_factory=user_factory) if expect_parse_exception: self.assertRaises(Exception, do_parse) else: node = do_parse() if expect_parse_exception: self.mox.VerifyAll() return None do_render = lambda: node.render(context) if expect_render_exception: self.assertRaises(Exception, do_render) else: actual_render_result = do_render() if expect_render_exception: self.mox.VerifyAll() return None self.mox.VerifyAll() self.assertTrue(actual_render_result == "" or actual_render_result == internal_render_result) in_group = (actual_render_result == internal_render_result) return in_group def doRenderExperiment(self, username, groupname, **kwargs): return self.doRenderExperimentToken( username, ("experiment", self.experiment.name, groupname), **kwargs) def testExperimentTag(self): found_test = False found_control = False for i in range(100): username = "******" % i in_test = self.doRenderExperiment(username, "test") found_test = found_test or in_test found_control = found_control or (not in_test) # make sure we get the same result from a second call self.assertEquals(in_test, self.doRenderExperiment(username, "test")) # make sure that the result of calling control is the opposite of # the result of calling test in_control = self.doRenderExperiment(username, "control") self.assertEquals(not in_test, in_control) self.assertTrue(found_control and found_test) def doRenderClientSideExperiment(self, context, username, experiment_name): parser = self.mox.CreateMockAnything() user_factory = self.mox.CreateMockAnything() token = self.mox.CreateMockAnything() token.split_contents().AndReturn(("clientsideexperiment", experiment_name)) user_factory.create_user(context).MultipleTimes().AndReturn( TestUser(username=username)) self.mox.ReplayAll() # HACK this is the only way to make a call optional user_factory.create_user(context) node = clientsideexperiment(parser, token, user_factory=user_factory) node.render(context) self.mox.VerifyAll() def testClientSideExperimentTag(self): for i in range(100): user = User(username="******" % i, email="*****@*****.**" % i) user.save() # create a context for a single "request" c = Context({"user": user}) self.doRenderClientSideExperiment(c, "user%i" % i, self.experiment.name) self.assertTrue("client_side_experiments" in c) self.doRenderClientSideExperiment(c, "user%i" % i, self.experiment.name) self.doRenderClientSideExperiment(c, "user%i" % i, self.other_experiment.name) self.doRenderClientSideExperiment(c, "user%i" % i, self.other_experiment.name) # determine what group the user was assigned to in each experiment experiment_partcipant = Participant.objects.get( user=user, experiment=self.experiment) other_experiment_partcipant = Participant.objects.get( user=user, experiment=self.other_experiment) # assert that exactly two entries were populated in the # 'client_side_experiments' dict in the context and that they are # equal to the names of our two experiments self.assertEquals(2, len(c['client_side_experiments'].keys())) self.assertTrue(self.experiment.name in c['client_side_experiments']) self.assertTrue(self.other_experiment.name in c['client_side_experiments']) # get the values that were put in the client_side_experiments dict group_name = c['client_side_experiments'][self.experiment.name] other_group_name = \ c['client_side_experiments'][self.other_experiment.name] # make sure that the values correctly represent the user's # enrollments group_id = experiment_partcipant.group other_group_id = other_experiment_partcipant.group self.assertEqual(group_id == Participant.TEST_GROUP, group_name == "test") self.assertEqual(group_id == Participant.CONTROL_GROUP, group_name == "control") self.assertEqual( other_group_id == Participant.TEST_GROUP, other_group_name == "test") self.assertEqual( other_group_id == Participant.CONTROL_GROUP, other_group_name == "control")