def setUp(self):
        self.experiment = Experiment(name="test_experiment")
        self.experiment.save()
        self.experiment.state = Experiment.ENABLED_STATE
        self.experiment.save()
        self.experiment.start_date = (self.experiment.start_date -
                                      timedelta(days=5))
        self.experiment.save()

        anonymous_visitor = AnonymousVisitor()
        anonymous_visitor.save()
        anonymous_participant = Participant(
            anonymous_visitor=anonymous_visitor,
            experiment=self.experiment,
            group=Participant.TEST_GROUP)
        anonymous_participant.save()
        anonymous_participant.enrollment_date = self.experiment.start_date
        anonymous_participant.save()

        self.other_experiment = Experiment(name="test_experiment2")
        self.other_experiment.save()
        self.other_experiment.state = Experiment.DISABLED_STATE
        self.other_experiment.save()
        self.other_experiment.start_date = (date.today() - timedelta(days=7))
        self.other_experiment.end_date = (date.today() - timedelta(days=3))
        self.other_experiment.save()
 def setUp(self):
     self.experiment = Experiment(name="test_experiment")
     self.experiment.save()
     self.experiment.state = Experiment.ENABLED_STATE
     self.experiment.save()
     self.experiment.start_date = (self.experiment.start_date -
                                   timedelta(days=5))
     self.experiment.save()
     
     anonymous_visitor = AnonymousVisitor()
     anonymous_visitor.save()
     anonymous_participant = Participant(anonymous_visitor=anonymous_visitor,
                                         experiment=self.experiment,
                                         group=Participant.TEST_GROUP)
     anonymous_participant.save()
     anonymous_participant.enrollment_date = self.experiment.start_date
     anonymous_participant.save()
     
     self.other_experiment = Experiment(name="test_experiment2")
     self.other_experiment.save()
     self.other_experiment.state = Experiment.DISABLED_STATE
     self.other_experiment.save()
     self.other_experiment.start_date = (date.today() -
                                   timedelta(days=7))
     self.other_experiment.end_date = (date.today() -
                                   timedelta(days=3))
     self.other_experiment.save()
 def testDisabledExperiment(self):
     # disabled test (unenrolled user)
     experiment = Experiment(name="disabled")
     experiment.save()
     user = TestUser(username="******")
     self.assertFalse(Experiment.test("disabled", user))
     self.assertTrue(Experiment.control("disabled", user))
 def setUp(self):
     self.experiment = Experiment(name="test_experiment")
     self.experiment.save()
     self.experiment.state = Experiment.ENABLED_STATE
     self.experiment.save()
     self.experiment.start_date = (self.experiment.start_date -
                                   timedelta(days=5))
     self.experiment.save()
Exemple #5
0
 def setUp(self):
     self.experiment = Experiment(name="test_experiment")
     self.experiment.save()
     self.experiment.state = Experiment.ENABLED_STATE
     self.experiment.save()
     
     self.other_experiment = Experiment(name="other_test_experiment")
     self.other_experiment.save()
     self.other_experiment.state = Experiment.ENABLED_STATE
     self.other_experiment.save()
     self.mox = mox.Mox()
 def setUp(self):
     staff_user = User(username="******", email="*****@*****.**",
                       is_staff=True)
     staff_user.save()
     staff_user.set_password("staff")
     staff_user.save()
     
     # self.assertTrue(self.client.login(username='******',
     #                                   password='******'))
     
     self.experiment1 = Experiment(name="experiment 1")
     self.experiment1.save()
     self.experiment1.state = Experiment.ENABLED_STATE
     self.experiment1.save()
     self.experiment1.start_date -= timedelta(days=5)
     self.experiment1.save()
     
     goal_types = [GoalType.objects.create(name='test_goal_%s' % i)
                   for i in range(3)]
     
     for i in range(1, 6):
         DailyEngagementReport.objects.create(date=days_ago(i),
                                              experiment=self.experiment1,
                                              control_score=3.2,
                                              test_score=2.3,
                                              control_group_size=3,
                                              test_group_size=5,
                                              confidence=93.2)
         conversion_report = DailyConversionReport.objects.create(
             date=days_ago(i),
             experiment=self.experiment1,
             overall_test_conversion=12,
             overall_control_conversion=9,
             test_group_size=39,
             control_group_size=27,
             confidence=87.4)
         for goal_type in goal_types:
             DailyConversionReportGoalData.objects.create(
                 report=conversion_report,
                 goal_type=goal_type,
                 test_conversion=11,
                 control_conversion=7,
                 confidence=45.3)
     
     self.experiment2 = Experiment(name="experiment 2")
     self.experiment2.save()
     
     self.experiment3 = Experiment(name="experiment 3")
     self.experiment3.save()
     self.experiment3.state = Experiment.ENABLED_STATE
     self.experiment3.save()
     self.experiment3.state = Experiment.PROMOTED_STATE
     self.experiment3.save()
 def testPromotedExperiment(self):
     # promoted test (unenrolled user)
     experiment = Experiment(name="promoted")
     experiment.save()
     experiment.state = Experiment.PROMOTED_STATE
     experiment.save()
     user = TestUser(username="******")
     self.assertTrue(Experiment.test("promoted", user))
     self.assertFalse(Experiment.control("promoted", user))
Exemple #8
0
    def render(self, context):
        user = self.get_user(context)
        should_render = False

        if self.group_name == "test":
            should_render = Experiment.test(self.experiment_name, user)
        elif self.group_name == "control":
            should_render = Experiment.control(self.experiment_name, user)
        else:
            raise Exception("Unknown Experiment group name : %s" %
                            self.group_name)

        if should_render:
            return self.node_list.render(context)
        else:
            return ""
 def render(self, context):
     user = self.get_user(context)
     should_render = False
     
     if self.group_name == "test":
         should_render = Experiment.test(self.experiment_name, user)
     elif self.group_name == "control":
         should_render = Experiment.control(self.experiment_name, user)
     else:
         raise Exception("Unknown Experiment group name : %s" %
                         self.group_name)
     
     if should_render:
         return self.node_list.render(context)
     else:
         return ""
    def testUnknownExperiment(self):
        # unknown test (unenrolled user)
        user = TestUser(username="******")
        
        # unit tests are always run in DEBUG=False by Django
        self.assertFalse(Experiment.test("undefined", user))
        self.assertTrue(Experiment.control("undefined", user))

        try:
            settings.DEBUG = True
            self.assertRaises(Exception,
                              lambda: Experiment.test("undefined", user))
            self.assertRaises(Exception,
                              lambda: Experiment.control("undefined", user))
        finally:
            settings.DEBUG = False
    def testUnknownExperiment(self):
        # unknown test (unenrolled user)
        user = TestUser(username="******")
        
        # unit tests are always run in DEBUG=False by Django
        self.assertFalse(Experiment.test("undefined", user))
        self.assertTrue(Experiment.control("undefined", user))

        try:
            settings.DEBUG = True
            self.assertRaises(Exception,
                              lambda: Experiment.test("undefined", user))
            self.assertRaises(Exception,
                              lambda: Experiment.control("undefined", user))
        finally:
            settings.DEBUG = False
 def testPromotedExperiment(self):
     # promoted test (unenrolled user)
     experiment = Experiment(name="promoted")
     experiment.save()
     experiment.state = Experiment.PROMOTED_STATE
     experiment.save()
     user = TestUser(username="******")
     self.assertTrue(Experiment.test("promoted", user))
     self.assertFalse(Experiment.control("promoted", user))
 def setUp(self):
     self.experiment = Experiment(name="test_experiment")
     self.experiment.save()
     self.experiment.state = Experiment.ENABLED_STATE
     self.experiment.save()
     self.experiment.start_date = (self.experiment.start_date -
                                   timedelta(days=5))
     self.experiment.save()
class TestManagement(TestCase):
    def setUp(self):
        self.experiment = Experiment(name="test_experiment")
        self.experiment.save()
        self.experiment.state = Experiment.ENABLED_STATE
        self.experiment.save()
        self.experiment.start_date = (self.experiment.start_date -
                                      timedelta(days=5))
        self.experiment.save()
    
    def testManageCommand(self):        
        with patch(settings, 'LEAN_ENGAGEMENT_CALCULATOR',
                   'django_lean.experiments.testsettings.SimpleEngagementCalculator'): 
            #make sure the manage.py command that generates daily stats work
            
            #Running with arguments should raise Exception
            self.assertRaises(CommandError,
                update_experiment_reports.Command().handle,
                "some", "args")
            
            #This is what manage.py will call
            self.runner = update_experiment_reports.Command().run_from_argv
            #Run the reports
            self.runner(['manage.py', 'update_experiment_reports'])
            
            #Make sure they were generated
            self.assertEquals(5, DailyEngagementReport.objects.filter(
                    experiment=self.experiment).count())
            self.assertEquals(5, DailyConversionReport.objects.filter(
                    experiment=self.experiment).count())
 def testDisabledExperiment(self):
     # disabled test (unenrolled user)
     experiment = Experiment(name="disabled")
     experiment.save()
     user = TestUser(username="******")
     self.assertFalse(Experiment.test("disabled", user))
     self.assertTrue(Experiment.control("disabled", user))
Exemple #16
0
    def testBotExclusion(self):
        experiment = Experiment(name="bot_experiment")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()

        user = TestUser(verified_human=False)
        participants_count = Participant.objects.all().count()
        in_test = Experiment.test(experiment.name, user)
        self.assertEquals(None, user.get_anonymous_id())
        self.assertEquals(participants_count,
                          Participant.objects.all().count())

        enrollments = user.get_added_enrollments()
        self.assertEquals(len(enrollments.keys()), 1)
        self.assertTrue(experiment.name in enrollments.keys())
Exemple #17
0
    def render(self, context):
        """
        Appends to a 'client_side_experiments' variable in the context. It
        will be the templates responsibility to render this list into the
        Javascript context.
        """
        if self.CONTEXT_KEY not in context:
            context[self.CONTEXT_KEY] = {}

        if self.experiment_name not in context[self.CONTEXT_KEY]:
            user = self.create_user(context)
            group = None

            if Experiment.test(self.experiment_name, user):
                group = "test"
            elif Experiment.control(self.experiment_name, user):
                group = "control"
            else:
                raise Exception("Unexpected test group for experiment %s" %
                                self.experiment_name)

            context[self.CONTEXT_KEY][self.experiment_name] = group
        return ""
 def render(self, context):
     """
     Appends to a 'client_side_experiments' variable in the context. It
     will be the templates responsibility to render this list into the
     Javascript context.
     """
     if self.CONTEXT_KEY not in context:
         context[self.CONTEXT_KEY]= {}
     
     if self.experiment_name not in context[self.CONTEXT_KEY]:
         user = self.create_user(context)
         group = None
         
         if Experiment.test(self.experiment_name, user):
             group = "test"
         elif Experiment.control(self.experiment_name, user):
             group = "control"
         else:
             raise Exception("Unexpected test group for experiment %s" %
                             self.experiment_name)
         
         context[self.CONTEXT_KEY][self.experiment_name] = group
     return ""
Exemple #19
0
 def testGroupSanity(self):
     experiment = Experiment(name="experiment")
     experiment.save()
     experiment.state = Experiment.ENABLED_STATE
     experiment.save()
     experiment_url = reverse('experiments.tests.views.experiment_test',
                              args=[experiment.name])
     for i in range(100):
         self.client = Client()
         response = self.client.get(experiment_url)
         self.assertEquals(response.status_code, 200)
         self.assertTrue(response.content.strip().lower() in ("test",
                                                              "control"))
Exemple #20
0
    def setUp(self):
        staff_user = User(username="******",
                          email="*****@*****.**",
                          is_staff=True)
        staff_user.save()
        staff_user.set_password("staff")
        staff_user.save()

        # self.assertTrue(self.client.login(username='******',
        #                                   password='******'))

        self.experiment1 = Experiment(name="experiment 1")
        self.experiment1.save()
        self.experiment1.state = Experiment.ENABLED_STATE
        self.experiment1.save()
        self.experiment1.start_date -= timedelta(days=5)
        self.experiment1.save()

        goal_types = [
            GoalType.objects.create(name='test_goal_%s' % i) for i in range(3)
        ]

        for i in range(1, 6):
            DailyEngagementReport.objects.create(date=days_ago(i),
                                                 experiment=self.experiment1,
                                                 control_score=3.2,
                                                 test_score=2.3,
                                                 control_group_size=3,
                                                 test_group_size=5,
                                                 confidence=93.2)
            conversion_report = DailyConversionReport.objects.create(
                date=days_ago(i),
                experiment=self.experiment1,
                overall_test_conversion=12,
                overall_control_conversion=9,
                test_group_size=39,
                control_group_size=27,
                confidence=87.4)
            for goal_type in goal_types:
                DailyConversionReportGoalData.objects.create(
                    report=conversion_report,
                    goal_type=goal_type,
                    test_conversion=11,
                    control_conversion=7,
                    confidence=45.3)

        self.experiment2 = Experiment(name="experiment 2")
        self.experiment2.save()

        self.experiment3 = Experiment(name="experiment 3")
        self.experiment3.save()
        self.experiment3.state = Experiment.ENABLED_STATE
        self.experiment3.save()
        self.experiment3.state = Experiment.PROMOTED_STATE
        self.experiment3.save()
 def testBotExclusion(self):
     experiment = Experiment(name="bot_experiment")
     experiment.save()
     experiment.state = Experiment.ENABLED_STATE
     experiment.save()
     
     user = TestUser(verified_human=False)
     participants_count = Participant.objects.all().count()
     in_test = Experiment.test(experiment.name, user)
     self.assertEquals(None, user.get_anonymous_id())
     self.assertEquals(participants_count, Participant.objects.all().count())
     
     enrollments = user.get_added_enrollments()
     self.assertEquals(len(enrollments.keys()), 1)
     self.assertTrue(experiment.name in enrollments.keys())
 def testGroupSanity(self):
     experiment = Experiment(name="experiment")
     experiment.save()
     experiment.state = Experiment.ENABLED_STATE
     experiment.save()
     experiment_url = reverse('experiments.tests.views.experiment_test',
                              args=[experiment.name])
     for i in range(100):
         self.client = Client()
         response = self.client.get(experiment_url)
         self.assertEqual(response.status_code, 200)
         self.assertTrue(response.content.strip().lower() in ("test",
                                                              "control"))
Exemple #23
0
class TestManagement(TestCase):
    def setUp(self):
        self.experiment = Experiment(name="test_experiment")
        self.experiment.save()
        self.experiment.state = Experiment.ENABLED_STATE
        self.experiment.save()
        self.experiment.start_date = (self.experiment.start_date -
                                      timedelta(days=5))
        self.experiment.save()

    def testManageCommand(self):
        with patch(
                settings, 'LEAN_ENGAGEMENT_CALCULATOR',
                'django_lean.experiments.testsettings.SimpleEngagementCalculator'
        ):
            #make sure the manage.py command that generates daily stats work

            #Running with arguments should raise Exception
            self.assertRaises(CommandError,
                              update_experiment_reports.Command().handle,
                              "some", "args")

            #This is what manage.py will call
            self.runner = update_experiment_reports.Command().run_from_argv
            #Run the reports
            self.runner(['manage.py', 'update_experiment_reports'])

            #Make sure they were generated
            self.assertEquals(
                5,
                DailyEngagementReport.objects.filter(
                    experiment=self.experiment).count())
            self.assertEquals(
                5,
                DailyConversionReport.objects.filter(
                    experiment=self.experiment).count())
Exemple #24
0
    def testVerifyHuman(self):
        experiment = Experiment(name="experiment")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()

        other_experiment = Experiment(name="other_experiment")
        other_experiment.save()
        other_experiment.state = Experiment.ENABLED_STATE
        other_experiment.save()

        self.client = Client()

        original_participants_count = Participant.objects.all().count()
        original_anonymous_visitors_count = AnonymousVisitor.objects.all(
        ).count()
        experiment_url = reverse('experiments.tests.views.experiment_test',
                                 args=[experiment.name])
        response = self.client.get(experiment_url)
        self.assertEquals(response.status_code, 200)

        self.assertEquals(original_participants_count,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count,
                          AnonymousVisitor.objects.all().count())

        confirm_human_url = reverse('experiments.views.confirm_human')
        response = self.client.get(confirm_human_url)
        self.assertEquals(response.status_code, 204)
        self.assertEquals(0, len(response.content))
        self.assertEquals(original_participants_count + 1,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())

        # calling it again does nothing
        response = self.client.get(confirm_human_url)
        self.assertEquals(response.status_code, 204)
        self.assertEquals(0, len(response.content))
        self.assertEquals(original_participants_count + 1,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())

        other_experiment_url = reverse(
            'experiments.tests.views.experiment_test',
            args=[other_experiment.name])
        response = self.client.get(other_experiment_url)
        self.assertEquals(response.status_code, 200)

        # a new participant is immediately created for the new experiment
        self.assertEquals(original_participants_count + 2,
                          Participant.objects.all().count())
        # the existing anonymous visitor is reused
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())

        # the new experiment does not cause storage of a temporary enrollment
        response = self.client.get(confirm_human_url)
        self.assertEquals(response.status_code, 204)
        self.assertEquals(0, len(response.content))
        self.assertEquals(original_participants_count + 2,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())
class TestExperimentViews(object):
    urls = 'experiments.tests.urls'
    
    def setUp(self):
        staff_user = User(username="******", email="*****@*****.**",
                          is_staff=True)
        staff_user.save()
        staff_user.set_password("staff")
        staff_user.save()
        
        # self.assertTrue(self.client.login(username='******',
        #                                   password='******'))
        
        self.experiment1 = Experiment(name="experiment 1")
        self.experiment1.save()
        self.experiment1.state = Experiment.ENABLED_STATE
        self.experiment1.save()
        self.experiment1.start_date -= timedelta(days=5)
        self.experiment1.save()
        
        goal_types = [GoalType.objects.create(name='test_goal_%s' % i)
                      for i in range(3)]
        
        for i in range(1, 6):
            DailyEngagementReport.objects.create(date=days_ago(i),
                                                 experiment=self.experiment1,
                                                 control_score=3.2,
                                                 test_score=2.3,
                                                 control_group_size=3,
                                                 test_group_size=5,
                                                 confidence=93.2)
            conversion_report = DailyConversionReport.objects.create(
                date=days_ago(i),
                experiment=self.experiment1,
                overall_test_conversion=12,
                overall_control_conversion=9,
                test_group_size=39,
                control_group_size=27,
                confidence=87.4)
            for goal_type in goal_types:
                DailyConversionReportGoalData.objects.create(
                    report=conversion_report,
                    goal_type=goal_type,
                    test_conversion=11,
                    control_conversion=7,
                    confidence=45.3)
        
        self.experiment2 = Experiment(name="experiment 2")
        self.experiment2.save()
        
        self.experiment3 = Experiment(name="experiment 3")
        self.experiment3.save()
        self.experiment3.state = Experiment.ENABLED_STATE
        self.experiment3.save()
        self.experiment3.state = Experiment.PROMOTED_STATE
        self.experiment3.save()
    
    def testListExperimentsView(self):
        url = reverse('experiments.views.list_experiments')
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        header, data = get_tables(response.content)[0]
        data.sort()
        self.assertEqual([u'Name', u'Status', u'Start Date', u'End Date'],
                          header)
        self.assertEqual([[u'experiment 1', u'Enabled',
                            unicode(self.experiment1.start_date), u'None'],
                           [u'experiment 2', u'Disabled', u'None', u'None'],
                           [u'experiment 3', u'Promoted',
                            unicode(self.experiment3.start_date),
                            unicode(self.experiment3.end_date)]],
                          data)
    
    def testAnonymousUserCannotAccessReports(self):
        self.client.logout()
        url = reverse('experiments.views.experiment_details',
                      args=['experiment 1'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)
        url = reverse('experiments.views.list_experiments')
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)
    
    def testRegularUserCannotAccessReports(self):
        self.client.logout()
        
        regular_user = User(username="******", email="*****@*****.**")
        regular_user.save()
        regular_user.set_password("joe")
        regular_user.save()
        
        self.assertTrue(self.client.login(username='******',
                                          password='******'))
        
        url = reverse('experiments.views.experiment_details',
                      args=['experiment 1'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)
        url = reverse('experiments.views.list_experiments')
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)
    
    def test404IfExperimentDoesntExist(self):
        url = reverse('experiments.views.experiment_details',
                      args=['inexistant experiment'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 404)
    
    def testExperimentDetailsView(self):
        url = reverse('experiments.views.experiment_details',
                      args=['experiment 1'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        tables = get_tables(response.content)
        experiment_properties = tables.pop(0)
        self.assertEqual([u'Property', u'Value'],
                          experiment_properties[0])
        self.assertEqual([[u'Experiment Name', u'experiment 1'],
                           [u'Status', u'Enabled'],
                           [u'Start Date',
                            unicode(self.experiment1.start_date)],
                           [u'End Date', u'None']],
                          experiment_properties[1])
        conversion_summary = tables.pop(0)
        self.assertEqual([u'&nbsp;', u'Control Group', u'Test Group',
                           u'Improvement', u'Confidence'],
                          conversion_summary[0])
        self.assertEqual([[u'Participants', u'27', u'39', u'&nbsp;',
                            u'&nbsp;'],
                           [u'test_goal_0', u'7 (25.9 %)', u'11 (28.2 %)',
                            u'8 %', u'45 %'],
                           [u'test_goal_1', u'7 (25.9 %)', u'11 (28.2 %)',
                            u'8 %', u'45 %'],
                           [u'test_goal_2', u'7 (25.9 %)', u'11 (28.2 %)',
                            u'8 %', u'45 %'],
                           [u'Any', u'9 (33.3 %)', u'12 (30.8 %)', u'-7 %',
                            u'87 %']],
                          conversion_summary[1])
        engagement_summary = tables.pop(0)
        self.assertEqual([u'&nbsp;', u'Control Group', u'Test Group',
                           u'Improvement', u'Confidence'],
                          engagement_summary[0])
        self.assertEqual([[u'Participants', u'3', u'5', u'&nbsp;', u'&nbsp;'],
                           [u'Engagement Score', u'3.2', u'2.3', u'-28 %',
                            u'93 %']],
                          engagement_summary[1])
        conversion_details = tables.pop(0)
        self.assertTrue(conversion_details[0])
        self.assertTrue(conversion_details[1])
        engagement_details = tables.pop(0)
        self.assertEqual([u'Report Date', u'Control Group Size',
                           u'Control Group Score', u'Test Group Size',
                           u'Test Group Score', u'Test Group Improvement',
                           u'Confidence'],
                          engagement_details[0])
        self.assertEqual([unicode(days_ago(1)), u'3', u'3.2', u'5', u'2.3',
                           u'-28 %', u'93 %'],
                          engagement_details[1][0])
        self.assertEqual([], tables)
    
    def testVerifyHuman(self):
        experiment = Experiment(name="experiment")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        
        other_experiment = Experiment(name="other_experiment")
        other_experiment.save()
        other_experiment.state = Experiment.ENABLED_STATE
        other_experiment.save()
        
        self.client = Client()
        
        original_participants_count = Participant.objects.all().count()
        original_anonymous_visitors_count = AnonymousVisitor.objects.all().count()
        experiment_url = reverse('experiments.tests.views.experiment_test',
                                 args=[experiment.name])
        response = self.client.get(experiment_url)
        self.assertEqual(response.status_code, 200)
        
        self.assertEqual(original_participants_count,
                          Participant.objects.all().count())
        self.assertEqual(original_anonymous_visitors_count,
                          AnonymousVisitor.objects.all().count())
        
        confirm_human_url = reverse('experiments.views.confirm_human')
        response = self.client.get(confirm_human_url)
        self.assertEqual(response.status_code, 204)
        self.assertEqual(0, len(response.content))
        self.assertEqual(original_participants_count+1,
                          Participant.objects.all().count())
        self.assertEqual(original_anonymous_visitors_count+1,
                          AnonymousVisitor.objects.all().count())
        
        # calling it again does nothing
        response = self.client.get(confirm_human_url)
        self.assertEqual(response.status_code, 204)
        self.assertEqual(0, len(response.content))
        self.assertEqual(original_participants_count+1,
                          Participant.objects.all().count())
        self.assertEqual(original_anonymous_visitors_count+1,
                          AnonymousVisitor.objects.all().count())
        
        other_experiment_url = reverse(
            'experiments.tests.views.experiment_test',
            args=[other_experiment.name])
        response = self.client.get(other_experiment_url)
        self.assertEqual(response.status_code, 200)
        
        # a new participant is immediately created for the new experiment
        self.assertEqual(original_participants_count + 2,
                          Participant.objects.all().count())
        # the existing anonymous visitor is reused
        self.assertEqual(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())
        
        # the new experiment does not cause storage of a temporary enrollment
        response = self.client.get(confirm_human_url)
        self.assertEqual(response.status_code, 204)
        self.assertEqual(0, len(response.content))
        self.assertEqual(original_participants_count+2,
                          Participant.objects.all().count())
        self.assertEqual(original_anonymous_visitors_count+1,
                          AnonymousVisitor.objects.all().count())
    
    def testGroupSanity(self):
        experiment = Experiment(name="experiment")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment_url = reverse('experiments.tests.views.experiment_test',
                                 args=[experiment.name])
        for i in range(100):
            self.client = Client()
            response = self.client.get(experiment_url)
            self.assertEqual(response.status_code, 200)
            self.assertTrue(response.content.strip().lower() in ("test",
                                                                 "control"))
    def testEnabledPromotedAndDisabledExperiment(self):
        # enabled test, new user (prove we get both results)
        experiment = Experiment(name="enabled")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        test_user = None
        control_user = None
        for i in range(100):
            user = TestUser(username="******" % i)
            in_test = Experiment.test("enabled", user)
            in_control = Experiment.control("enabled", user)
            self.assertNotEquals(in_test, in_control)
            self.assertEquals(in_test, Experiment.test("enabled", user))
            self.assertEquals(in_control, Experiment.control("enabled", user))
            if in_test:
                test_user = user
            if in_control:
                control_user = user

        self.assertNotEquals(None, test_user)
        self.assertNotEquals(None, control_user)
        self.assertNotEquals(test_user, control_user)

        # promoted test with an enrolled user
        # (prove we get both results, then that all users are in test)
        experiment.state = Experiment.PROMOTED_STATE
        experiment.save()

        self.assertFalse(Experiment.control("enabled", control_user))
        self.assertFalse(Experiment.control("enabled", test_user))
        self.assertTrue(Experiment.test("enabled", control_user))
        self.assertTrue(Experiment.test("enabled", test_user))

        # disabled test with an enrolled user
        # (prove we get both results, then that all users are in control)
        experiment.state = Experiment.DISABLED_STATE
        experiment.save()

        self.assertTrue(Experiment.control("enabled", control_user))
        self.assertTrue(Experiment.control("enabled", test_user))
        self.assertFalse(Experiment.test("enabled", control_user))
        self.assertFalse(Experiment.test("enabled", test_user))
 def testPngResponse(self):
     experiment = Experiment(name="test-experiment")
     experiment.save()
     experiment.state = Experiment.ENABLED_STATE
     experiment.save()
     
     goal_type = GoalType(name='test-goal')
     goal_type.save()
     
     experiment_url = reverse("django_lean.experiments.tests.views.experiment_test",
                              args=[experiment.name])
     confirm_human_url = reverse("django_lean.experiments.views.confirm_human")
     
     client = Client()
     
     # we can call with invalid or inexisting names, the response is the same
     self.call_goal(client, '')
     self.call_goal(client, 'unknown-goal')
     
     # this is an anonymous visitor not enrolled in an experiment,
     # so no records should be created
     self.call_goal(client, goal_type.name)
     
     self.assertEqual(0, GoalRecord.objects.filter(goal_type=goal_type).count())
     
     nb_anonymous_visitors = AnonymousVisitor.objects.count()
     # force the user to be a verified human
     response = client.get(confirm_human_url)
     self.assertEqual(response.status_code, 204)
     
     # force the anonymous visitor to be enrolled in an experiment
     response = client.get(experiment_url)
     self.assertEqual(response.status_code, 200)
     self.assertEqual(nb_anonymous_visitors + 1,
                       AnonymousVisitor.objects.count())
     client.get('/test-experiment/%s' % experiment.name)
     self.assertEqual(nb_anonymous_visitors + 1,
                       AnonymousVisitor.objects.count())
     
     # now call an existing goal again - it should be recorded
     self.call_goal(client, goal_type.name)
     self.assertEqual(1, GoalRecord.objects.filter(goal_type=goal_type).count())
     
     # should be recorded again
     self.call_goal(client, goal_type.name)
     self.assertEqual(2, GoalRecord.objects.filter(goal_type=goal_type).count())
     
     # validate that both of the records have the same anonymous_visitor
     two_goal_records = GoalRecord.objects.filter(goal_type=goal_type)
     self.assertEqual(two_goal_records[0].anonymous_visitor,
                       two_goal_records[1].anonymous_visitor)
     
     # try it with a registered user
     client = Client()
     user = User(username="******", email="*****@*****.**")
     user.set_password("password")
     user.save()
     
     response = client.login(username=user.username, password='******')
     self.assertTrue(response)
     
     # force the registered user to be enrolled in an experiment
     client.get('/test-experiment/%s' % experiment.name)
     
     self.call_goal(client, goal_type.name)
     # since the user was registered, no new records should be created
     self.assertEqual(2, GoalRecord.objects.filter(goal_type=goal_type).count())
 def testParticipantEnrollment(self):
     experiment1 = Experiment(name="Experiment 1")
     experiment1.save()
     experiment1.state = Experiment.ENABLED_STATE
     experiment1.save()
     experiment2 = Experiment(name="Experiment 2")
     experiment2.save()
     experiment2.state = Experiment.ENABLED_STATE
     experiment2.save()
     
     num_control1 = 0
     num_test1 = 0
     num_control2 = 0
     num_test2 = 0
     
     for i in range(1000):
         username="******" % i
         in_test1 = Experiment.test(experiment1.name, TestUser(username=username))
         self.assertEquals(in_test1, not Experiment.control(experiment1.name,
                                                            TestUser(username=username)))
         if in_test1:
             num_test1 += 1
         else:
             num_control1 += 1
         
         in_test2 = not Experiment.control(experiment2.name, TestUser(username=username))
         self.assertEquals(in_test2, Experiment.test(experiment2.name,
                                                     TestUser(username=username)))
         
         if in_test2:
             num_test2 += 1
         else:
             num_control2 += 1
     
     self.assert_(num_control1 > 400)
     self.assert_(num_control2 > 400)
     self.assert_(num_test1 > 400)
     self.assert_(num_test2 > 400)
 def testParticipants(self):
     user1 = User(username="******", email="*****@*****.**")
     user1.save()
     user2 = User(username="******", email="*****@*****.**")
     user2.save()
     user3 = User(username="******", email="*****@*****.**")
     user3.save()
     
     anonymous_visitor1 = AnonymousVisitor()
     anonymous_visitor1.save()
     anonymous_visitor2 = AnonymousVisitor()
     anonymous_visitor2.save()
     anonymous_visitor3 = AnonymousVisitor()
     anonymous_visitor3.save()
     
     experiment1 = Experiment(name="Experiment 1")
     experiment1.save()
     experiment2 = Experiment(name="Experiment 2")
     experiment2.save()
     
     participant11 = Participant(user=user1,
                                 experiment= experiment1,
                                 group=Participant.CONTROL_GROUP)
     participant11.save()
     participant12 = Participant(user=user1,
                                 experiment= experiment2,
                                 group=Participant.TEST_GROUP)
     participant12.save()
     participant21 = Participant(user=user2,
                                 experiment= experiment1,
                                 group=Participant.CONTROL_GROUP)
     participant21.save()
     participant22 = Participant(user=user2,
                                 experiment= experiment2,
                                 group=Participant.CONTROL_GROUP)
     participant22.save()
     
     self.assertRaises(Exception,
                       lambda: Participant(user=user2,
                                           experiment= experiment2,
                                           group=Participant.TEST_GROUP).save())
     
     # create anonymous participants
     participant31 = Participant(anonymous_visitor=anonymous_visitor1,
                                 experiment=experiment1,
                                 group=Participant.CONTROL_GROUP)
     participant31.save()
     participant32 = Participant(anonymous_visitor=anonymous_visitor1,
                                 experiment=experiment2,
                                 group=Participant.TEST_GROUP)
     participant32.save()
     participant42 = Participant(anonymous_visitor=anonymous_visitor2,
                                 experiment=experiment2,
                                 group=Participant.CONTROL_GROUP)
     participant42.save()
     
     self.assertRaises(Exception,
                       lambda: Participant(anonymous_visitor=anonymous_visitor1,
                                           experiment=experiment1,
                                           group=Participant.TEST_GROUP).save())
     
     self.assertRaises(Exception,
                       lambda: Participant(experiment=experiment1,
                                           group=Participant.TEST_GROUP).save())
     
     self.assertRaises(Exception,
                       lambda: Participant(user=user3,
                                           anonymous_visitor=anonymous_visitor3,
                                           experiment=experiment1,
                                           group=Participant.TEST_GROUP))
     self.assertRaises(Exception,
                       lambda: Participant(user=user3,
                                           experiment=experiment1).save())
     
     self.assertNotEquals(participant11.enrollment_date, None)
     
     participant11.enrollment_date = None
     self.assertRaises(Exception,
                       lambda: participant11.save())
     
     participant12.group = None
     self.assertRaises(Exception,
                       lambda: participant12.save())
Exemple #30
0
    def testPngResponse(self):
        experiment = Experiment(name="test-experiment")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()

        goal_type = GoalType(name='test-goal')
        goal_type.save()

        experiment_url = reverse(
            "django_lean.experiments.tests.views.experiment_test",
            args=[experiment.name])
        confirm_human_url = reverse(
            "django_lean.experiments.views.confirm_human")

        client = Client()

        # we can call with invalid or inexisting names, the response is the same
        self.call_goal(client, '')
        self.call_goal(client, 'unknown-goal')

        # this is an anonymous visitor not enrolled in an experiment,
        # so no records should be created
        self.call_goal(client, goal_type.name)

        self.assertEquals(
            0,
            GoalRecord.objects.filter(goal_type=goal_type).count())

        nb_anonymous_visitors = AnonymousVisitor.objects.count()
        # force the user to be a verified human
        response = client.get(confirm_human_url)
        self.assertEquals(response.status_code, 204)

        # force the anonymous visitor to be enrolled in an experiment
        response = client.get(experiment_url)
        self.assertEquals(response.status_code, 200)
        self.assertEquals(nb_anonymous_visitors + 1,
                          AnonymousVisitor.objects.count())
        client.get('/test-experiment/%s' % experiment.name)
        self.assertEquals(nb_anonymous_visitors + 1,
                          AnonymousVisitor.objects.count())

        # now call an existing goal again - it should be recorded
        self.call_goal(client, goal_type.name)
        self.assertEquals(
            1,
            GoalRecord.objects.filter(goal_type=goal_type).count())

        # should be recorded again
        self.call_goal(client, goal_type.name)
        self.assertEquals(
            2,
            GoalRecord.objects.filter(goal_type=goal_type).count())

        # validate that both of the records have the same anonymous_visitor
        two_goal_records = GoalRecord.objects.filter(goal_type=goal_type)
        self.assertEquals(two_goal_records[0].anonymous_visitor,
                          two_goal_records[1].anonymous_visitor)

        # try it with a registered user
        client = Client()
        user = User(username="******", email="*****@*****.**")
        user.set_password("password")
        user.save()

        response = client.login(username=user.username, password='******')
        self.assertTrue(response)

        # force the registered user to be enrolled in an experiment
        client.get('/test-experiment/%s' % experiment.name)

        self.call_goal(client, goal_type.name)
        # since the user was registered, no new records should be created
        self.assertEquals(
            2,
            GoalRecord.objects.filter(goal_type=goal_type).count())
    def testConversionReportGenerator(self):
        days = [
            datetime.combine(date.today() + timedelta(days=i), time(hour=12))
            for i in range(-7, 0)
        ]

        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = days[2].date()
        experiment.save()

        other_experiment = Experiment(name="experiment2")
        other_experiment.save()
        other_experiment.state = Experiment.DISABLED_STATE
        other_experiment.save()
        other_experiment.start_date = days[0].date()
        other_experiment.end_date = days[4].date()
        other_experiment.save()

        goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]

        # experiment starts on days[2]
        # other experiment starts on days[0]

        mocker = mox.Mox()
        finder = mocker.CreateMockAnything()
        calculator = mocker.CreateMockAnything()

        default_data = {
            Participant.TEST_GROUP: {
                "count": 110,
                "conversions": [23, 12, 9]
            },
            Participant.CONTROL_GROUP: {
                "count": 130,
                "conversions": [12, 47, 5]
            }
        }
        day_2_data = {
            Participant.TEST_GROUP: {
                "count": 12,
                "conversions": [0, 2, 3]
            },
            Participant.CONTROL_GROUP: {
                "count": 7,
                "conversions": [1, 0, 3]
            }
        }
        day_3_data = {
            Participant.TEST_GROUP: {
                "count": 5,
                "conversions": [1, 0, 3]
            },
            Participant.CONTROL_GROUP: {
                "count": 12,
                "conversions": [0, 0, 0]
            }
        }
        day_4_data = {
            Participant.TEST_GROUP: {
                "count": 0,
                "conversions": [0, 0, 0]
            },
            Participant.CONTROL_GROUP: {
                "count": 25,
                "conversions": [2, 3, 7]
            }
        }

        for day in days[2:7]:
            data = default_data
            if day == days[2]:
                data = day_2_data
            if day == days[3]:
                data = day_3_data
            if day == days[4]:
                data = day_4_data

            for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
                mock_participants = mocker.CreateMockAnything()
                finder(group, experiment,
                       day.date()).InAnyOrder().AndReturn(mock_participants)
                mock_participants.count().MultipleTimes().AndReturn(
                    data[group]["count"])
                for goal_type in goal_types:
                    calculator(goal_type, mock_participants,
                               day.date()).InAnyOrder().AndReturn(
                                   data[group]["conversions"][int(
                                       goal_type.name)])
                calculator(None, mock_participants,
                           day.date()).InAnyOrder().AndReturn(
                               sum(data[group]["conversions"]))
        mocker.ReplayAll()

        for d in days[2:7]:
            ConversionReportGenerator(
                calculator, finder).generate_daily_report_for_experiment(
                    experiment, d.date())

        results = DailyConversionReport.objects.filter(
            experiment=experiment).order_by('-date')

        mocker.VerifyAll()

        self.assertEquals(results.count(), 5)
        report_days = [d.date() for d in days[2:7]]
        for i in range(5):
            self.assertEquals(results[i].date, report_days[4 - i])

        # Day 2
        self.assertEquals(12, results[4].test_group_size)
        self.assertEquals(7, results[4].control_group_size)
        self.assertEquals(5, results[4].overall_test_conversion)
        self.assertEquals(4, results[4].overall_control_conversion)

        day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4], goal_type=goal_types[0])[0].test_conversion
        self.assertEquals(0, day_2_goal_4_test_conversion)

        day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4], goal_type=goal_types[2])[0].control_conversion
        self.assertEquals(3, day_2_goal_2_control_conversion)

        # Day 3
        self.assertEquals(5, results[3].test_group_size)
        # Day 4
        self.assertEquals(0, results[2].test_group_size)
        self.assertEquals(None, results[2].confidence)
        day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[2], goal_type=goal_types[0])[0].confidence
        self.assertEquals(None, day_4_goal_1_confidence)
        # Day 5
        day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[1], goal_type=goal_types[0])[0].confidence
        self.assertAlmostEqual(98.935467172597029,
                               day_5_goal_0_confidence,
                               places=6)
 def testGetConversionData(self):
     days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
             for i in range(-7, 0)]
     
     yesterday = date.today() - timedelta(days=1)
     experiment = Experiment(name="experiment1")
     experiment.save()
     experiment.state = Experiment.ENABLED_STATE
     experiment.save()
     experiment.start_date = yesterday
     experiment.save()
     
     goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
     
     report = DailyConversionReport.objects.create(experiment=experiment,
                                          date=yesterday,
                                          overall_test_conversion=17,
                                          overall_control_conversion=12,
                                          test_group_size=139,
                                          control_group_size=142,
                                          confidence=87.3)
     
     DailyConversionReportGoalData.objects.create(report=report,
                                                  goal_type=goal_types[0],
                                                  test_conversion=11,
                                                  control_conversion=0,
                                                  confidence=65.3)
     DailyConversionReportGoalData.objects.create(report=report,
                                                  goal_type=goal_types[1],
                                                  test_conversion=0,
                                                  control_conversion=21,
                                                  confidence=None)
     DailyConversionReportGoalData.objects.create(report=report,
                                                  goal_type=goal_types[2],
                                                  test_conversion=23,
                                                  control_conversion=21,
                                                  confidence=100)
     
     data = get_conversion_data(experiment, yesterday)
     
     self.assertEqual(data['date'], yesterday)
     self.assertTrue("totals" in data)
     self.assertTrue("goal_types" in data)
     self.assertEqual(4, len(data["goal_types"].keys()))
     
     for goal_type in goal_types[0:3]:
         self.assertTrue(goal_type.name in data["goal_types"])
         
         goal_type_data = data["goal_types"][goal_type.name]
         self.assertTrue("test_count" in goal_type_data)
         self.assertTrue("control_count" in goal_type_data)
         self.assertTrue("test_rate" in goal_type_data)
         self.assertTrue("control_rate" in goal_type_data)
         self.assertTrue("improvement" in goal_type_data)
         self.assertTrue("confidence" in goal_type_data)
     
     self.assertEqual(None, data["goal_types"][goal_types[3].name])
     
     self.assertEqual(139, data["test_group_size"])
     self.assertEqual(142, data["control_group_size"])
     
     totals = data["totals"]
     
     expected_test_rate = 17. / 139. * 100.
     expected_control_rate = 12. / 142. * 100.
     expected_improvement = (expected_test_rate - expected_control_rate) / expected_control_rate * 100.
     
     self.assertAlmostEqual(expected_test_rate, totals["test_rate"])
     self.assertAlmostEqual(expected_control_rate, totals["control_rate"])
     self.assertAlmostEqual(expected_improvement, totals["improvement"])
     self.assertAlmostEqual(87.3, totals["confidence"])
     self.assertEqual(17, totals["test_count"])
     self.assertEqual(12, totals["control_count"])
     
     self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_rate"])
     self.assertAlmostEqual(11./139*100., data["goal_types"][goal_types[0].name]["test_rate"])
     self.assertEqual(None, data["goal_types"][goal_types[0].name]["improvement"])
     self.assertAlmostEqual(65.3, data["goal_types"][goal_types[0].name]["confidence"])
     self.assertEqual(11, data["goal_types"][goal_types[0].name]["test_count"])
     self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_count"])
     
     self.assertAlmostEqual(21./142*100., data["goal_types"][goal_types[1].name]["control_rate"])
     self.assertEqual(None, data["goal_types"][goal_types[1].name]["confidence"])
     self.assertEqual(None, data["goal_types"][goal_types[1].name]["improvement"])
     
     self.assertAlmostEqual((23./139-21./142)/(21./142)*100.,
                             data["goal_types"][goal_types[2].name]["improvement"])
class TestDailyReports(TestCase):
    def setUp(self):
        self.experiment = Experiment(name="test_experiment")
        self.experiment.save()
        self.experiment.state = Experiment.ENABLED_STATE
        self.experiment.save()
        self.experiment.start_date = (self.experiment.start_date -
                                      timedelta(days=5))
        self.experiment.save()

        anonymous_visitor = AnonymousVisitor()
        anonymous_visitor.save()
        anonymous_participant = Participant(
            anonymous_visitor=anonymous_visitor,
            experiment=self.experiment,
            group=Participant.TEST_GROUP)
        anonymous_participant.save()
        anonymous_participant.enrollment_date = self.experiment.start_date
        anonymous_participant.save()

        self.other_experiment = Experiment(name="test_experiment2")
        self.other_experiment.save()
        self.other_experiment.state = Experiment.DISABLED_STATE
        self.other_experiment.save()
        self.other_experiment.start_date = (date.today() - timedelta(days=7))
        self.other_experiment.end_date = (date.today() - timedelta(days=3))
        self.other_experiment.save()

    def testDailyEngagementReport(self):
        users_test = []
        users_control = []

        num_control1 = 0
        num_test1 = 0
        num_control2 = 0
        num_test2 = 0

        #create users
        for i in range(5):
            users_control.append(
                create_user_in_group(self.experiment, i,
                                     Participant.CONTROL_GROUP,
                                     date.today() - timedelta(days=i)))
            users_test.append(
                create_user_in_group(self.experiment, i,
                                     Participant.TEST_GROUP,
                                     date.today() - timedelta(days=i)))

        # users_<test|control>[0] were enrolled today, [1] 1 day ago, etc.

        report_date = date.today() - timedelta(days=1)
        expected_engagement_score_calls = {
            (users_test[1], date.today() - timedelta(days=1), report_date):
            3.2,
            (users_test[2], date.today() - timedelta(days=2), report_date):
            2.5,
            (users_test[3], date.today() - timedelta(days=3), report_date):
            4.1,
            (users_test[4], date.today() - timedelta(days=4), report_date): 0,
            (users_control[1], date.today() - timedelta(days=1), report_date):
            0,
            (users_control[2], date.today() - timedelta(days=2), report_date):
            0,
            (users_control[3], date.today() - timedelta(days=3), report_date):
            0,
            (users_control[4], date.today() - timedelta(days=4), report_date):
            0
        }

        test_case = self

        class EngagementScoreCalculatorStub(object):
            def calculate_user_engagement_score(self, user, start_date,
                                                end_date):
                test_case.assertNotEquals(user, None)
                test_case.assertTrue(
                    expected_engagement_score_calls.has_key(
                        (user, start_date, end_date)))
                return expected_engagement_score_calls[(user, start_date,
                                                        end_date)]

        (EngagementReportGenerator(EngagementScoreCalculatorStub()).
         generate_daily_report_for_experiment(self.experiment, report_date))

        experiment_report = DailyEngagementReport.objects.get(
            experiment=self.experiment, date=report_date)
        self.assertAlmostEqual((3.2 + 2.5 + 4.1 + 0) / 4.0,
                               experiment_report.test_score)
        self.assertAlmostEqual(0.0, experiment_report.control_score)
        self.assertEquals(4, experiment_report.test_group_size)
        self.assertEquals(4, experiment_report.control_group_size)
        self.assertAlmostEqual(96.819293337188498,
                               experiment_report.confidence)

    def testZeroParticipantExperiment(self):
        mocker = mox.Mox()
        engagement_calculator = mocker.CreateMockAnything()
        mocker.ReplayAll()

        report_date = date.today()
        EngagementReportGenerator(
            engagement_score_calculator=engagement_calculator
        ).generate_daily_report_for_experiment(self.other_experiment,
                                               report_date)

        experiment_report = DailyEngagementReport.objects.get(
            experiment=self.other_experiment, date=report_date)

        mocker.VerifyAll()

        self.assertEquals(None, experiment_report.test_score)
        self.assertEquals(None, experiment_report.control_score)
        self.assertEquals(0, experiment_report.test_group_size)
        self.assertEquals(0, experiment_report.control_group_size)

    def testGenerateAllDailyEngagementReports(self):
        class DummyEngagementCalculator(object):
            def calculate_user_engagement_score(self, user, start_date,
                                                end_date):
                return 7

        engagement_report_generator = EngagementReportGenerator(
            engagement_score_calculator=DummyEngagementCalculator())
        engagement_report_generator.generate_daily_report_for_experiment(
            self.experiment,
            date.today() - timedelta(days=2))
        engagement_report_generator.generate_daily_report_for_experiment(
            self.experiment,
            date.today() - timedelta(days=3))
        engagement_report_generator.generate_all_daily_reports()
        DailyEngagementReport.objects.get(experiment=self.experiment,
                                          date=date.today() -
                                          timedelta(days=1))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                                          date=date.today() -
                                          timedelta(days=2))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                                          date=date.today() -
                                          timedelta(days=3))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                                          date=date.today() -
                                          timedelta(days=4))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                                          date=date.today() -
                                          timedelta(days=5))
        self.assertEquals(
            5,
            DailyEngagementReport.objects.filter(
                experiment=self.experiment).count())
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                                          date=date.today() -
                                          timedelta(days=3))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                                          date=date.today() -
                                          timedelta(days=4))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                                          date=date.today() -
                                          timedelta(days=5))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                                          date=date.today() -
                                          timedelta(days=6))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                                          date=date.today() -
                                          timedelta(days=7))

        self.assertEquals(
            5,
            DailyEngagementReport.objects.filter(
                experiment=self.other_experiment).count())

    def create_goal_record(self, record_datetime, anonymous_visitor,
                           goal_type):
        record = GoalRecord.objects.create(anonymous_visitor=anonymous_visitor,
                                           goal_type=goal_type)
        record.created = record_datetime
        record.save()

    def create_participant(self, anonymous_visitor, experiment,
                           enrollment_date, group):
        participant = Participant.objects.create(
            anonymous_visitor=anonymous_visitor,
            experiment=experiment,
            group=group)
        participant.enrollment_date = enrollment_date
        participant.save()
        return participant

    def testParticipantConversionCalculator(self):
        goal_types = [GoalType.objects.create(name=str(i)) for i in range(3)]
        anonymous_visitor = AnonymousVisitor.objects.create()
        participant = self.create_participant(
            anonymous_visitor=anonymous_visitor,
            experiment=self.experiment,
            enrollment_date=self.experiment.start_date + timedelta(days=2),
            group=Participant.TEST_GROUP)

        days = [
            datetime.combine(self.experiment.start_date + timedelta(days=i),
                             time(hour=12)) for i in range(5)
        ]

        nb_goal_records = GoalRecord.objects.all().count()

        self.create_goal_record(days[0], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[0], anonymous_visitor, goal_types[1])
        self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[2], anonymous_visitor, goal_types[1])
        self.create_goal_record(days[3], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[4], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[4], anonymous_visitor, goal_types[0])

        self.assertEquals(nb_goal_records + 8,
                          GoalRecord.objects.all().count())

        # wasn't enrolled yet!
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[0],
                                             days[0]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[1],
                                             days[0]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[2],
                                             days[0]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, None, days[0]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[0],
                                             days[1]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[1],
                                             days[1]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[2],
                                             days[1]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, None, days[1]), 0)

        # now enrolled
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[0],
                                             days[2]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[1],
                                             days[2]), 1)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[2],
                                             days[2]), 0)
        # "any" is one
        self.assertEquals(
            calculate_participant_conversion(participant, None, days[2]), 1)

        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[0],
                                             days[3]), 1)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[1],
                                             days[3]), 1)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[2],
                                             days[3]), 0)
        # "any" is one, even though two different goals were achieved
        self.assertEquals(
            calculate_participant_conversion(participant, None, days[3]), 1)

        # there were three conversions on this day for goal 0, but we only count the first!
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[0],
                                             days[4]), 1)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[1],
                                             days[4]), 1)
        self.assertEquals(
            calculate_participant_conversion(participant, goal_types[2],
                                             days[4]), 0)
        self.assertEquals(
            calculate_participant_conversion(participant, None, days[4]), 1)

    def testGoalTypeConversionCalculator(self):
        mocker = mox.Mox()
        participants = [
            mocker.CreateMockAnything(),
            mocker.CreateMockAnything(),
            mocker.CreateMockAnything()
        ]
        goal_type = mocker.CreateMockAnything()
        report_date = mocker.CreateMockAnything()
        participant_conversion_calculator = mocker.CreateMockAnything()

        participant_conversion_calculator(
            participants[0], goal_type, report_date).InAnyOrder().AndReturn(1)
        participant_conversion_calculator(
            participants[1], goal_type, report_date).InAnyOrder().AndReturn(0)
        participant_conversion_calculator(
            participants[2], goal_type, report_date).InAnyOrder().AndReturn(1)

        mocker.ReplayAll()

        self.assertEquals(
            2,
            calculate_goal_type_conversion(goal_type, participants,
                                           report_date,
                                           participant_conversion_calculator))

        mocker.VerifyAll()

    def testExperimentGroupParticipantFinder(self):
        days = [
            datetime.combine(date.today() + timedelta(days=i), time(hour=12))
            for i in range(-7, 0)
        ]

        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = days[2].date()
        experiment.save()

        other_experiment = Experiment(name="experiment2")
        other_experiment.save()
        other_experiment.state = Experiment.DISABLED_STATE
        other_experiment.save()
        other_experiment.start_date = days[0].date()
        other_experiment.end_date = days[4].date()
        other_experiment.save()

        anonymous_visitors = [
            AnonymousVisitor.objects.create() for i in range(10)
        ]

        experiment_participant_groups = [
            [
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[0],
                    experiment=experiment,
                    enrollment_date=days[2],
                    group=Participant.TEST_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[1],
                    experiment=experiment,
                    enrollment_date=days[2],
                    group=Participant.CONTROL_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[3],
                    experiment=experiment,
                    enrollment_date=days[3],
                    group=Participant.TEST_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[4],
                    experiment=experiment,
                    enrollment_date=days[4],
                    group=Participant.CONTROL_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[6],
                    experiment=experiment,
                    enrollment_date=days[6],
                    group=Participant.TEST_GROUP)
            ],
            [
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[0],
                    experiment=other_experiment,
                    enrollment_date=days[0],
                    group=Participant.CONTROL_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[2],
                    experiment=other_experiment,
                    enrollment_date=days[0],
                    group=Participant.TEST_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[5],
                    experiment=other_experiment,
                    enrollment_date=days[4],
                    group=Participant.TEST_GROUP)
            ]
        ]

        ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP,
                                                     experiment, days[2])
        ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
        self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
        self.assertEquals(1, len(ex1day2visitors))

        ex1day4test = find_experiment_group_participants(
            Participant.TEST_GROUP, experiment, days[4])
        ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
        self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
        self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
        self.assertEquals(2, len(ex1day4testvisitors))

        ex1day4control = find_experiment_group_participants(
            Participant.CONTROL_GROUP, experiment, days[4])
        ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
        self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
        self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
        self.assertEquals(2, len(ex1day4controlvisitors))

        ex2day5test = find_experiment_group_participants(
            Participant.TEST_GROUP, other_experiment, days[5])
        ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
        self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
        self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
        self.assertEquals(2, len(ex2day5testvisitors))

    def testGetConversionData(self):
        days = [
            datetime.combine(date.today() + timedelta(days=i), time(hour=12))
            for i in range(-7, 0)
        ]

        yesterday = date.today() - timedelta(days=1)
        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = yesterday
        experiment.save()

        goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]

        report = DailyConversionReport.objects.create(
            experiment=experiment,
            date=yesterday,
            overall_test_conversion=17,
            overall_control_conversion=12,
            test_group_size=139,
            control_group_size=142,
            confidence=87.3)

        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[0],
                                                     test_conversion=11,
                                                     control_conversion=0,
                                                     confidence=65.3)
        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[1],
                                                     test_conversion=0,
                                                     control_conversion=21,
                                                     confidence=None)
        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[2],
                                                     test_conversion=23,
                                                     control_conversion=21,
                                                     confidence=100)

        data = get_conversion_data(experiment, yesterday)

        self.assertEquals(data['date'], yesterday)
        self.assertTrue("totals" in data)
        self.assertTrue("goal_types" in data)
        self.assertEquals(4, len(data["goal_types"].keys()))

        for goal_type in goal_types[0:3]:
            self.assertTrue(goal_type.name in data["goal_types"])

            goal_type_data = data["goal_types"][goal_type.name]
            self.assertTrue("test_count" in goal_type_data)
            self.assertTrue("control_count" in goal_type_data)
            self.assertTrue("test_rate" in goal_type_data)
            self.assertTrue("control_rate" in goal_type_data)
            self.assertTrue("improvement" in goal_type_data)
            self.assertTrue("confidence" in goal_type_data)

        self.assertEquals(None, data["goal_types"][goal_types[3].name])

        self.assertEquals(139, data["test_group_size"])
        self.assertEquals(142, data["control_group_size"])

        totals = data["totals"]

        expected_test_rate = 17. / 139. * 100.
        expected_control_rate = 12. / 142. * 100.
        expected_improvement = (expected_test_rate - expected_control_rate
                                ) / expected_control_rate * 100.

        self.assertAlmostEquals(expected_test_rate, totals["test_rate"])
        self.assertAlmostEquals(expected_control_rate, totals["control_rate"])
        self.assertAlmostEquals(expected_improvement, totals["improvement"])
        self.assertAlmostEquals(87.3, totals["confidence"])
        self.assertEquals(17, totals["test_count"])
        self.assertEquals(12, totals["control_count"])

        self.assertEquals(
            0, data["goal_types"][goal_types[0].name]["control_rate"])
        self.assertAlmostEquals(
            11. / 139 * 100.,
            data["goal_types"][goal_types[0].name]["test_rate"])
        self.assertEquals(
            None, data["goal_types"][goal_types[0].name]["improvement"])
        self.assertAlmostEquals(
            65.3, data["goal_types"][goal_types[0].name]["confidence"])
        self.assertEquals(11,
                          data["goal_types"][goal_types[0].name]["test_count"])
        self.assertEquals(
            0, data["goal_types"][goal_types[0].name]["control_count"])

        self.assertAlmostEquals(
            21. / 142 * 100.,
            data["goal_types"][goal_types[1].name]["control_rate"])
        self.assertEquals(None,
                          data["goal_types"][goal_types[1].name]["confidence"])
        self.assertEquals(
            None, data["goal_types"][goal_types[1].name]["improvement"])

        self.assertAlmostEquals(
            (23. / 139 - 21. / 142) / (21. / 142) * 100.,
            data["goal_types"][goal_types[2].name]["improvement"])

#TODO test with zero participants and check rate == None

#TODO sometimes confidence cannot be calculated and must return None. Add a test to verify this.

    def testConversionReportGenerator(self):
        days = [
            datetime.combine(date.today() + timedelta(days=i), time(hour=12))
            for i in range(-7, 0)
        ]

        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = days[2].date()
        experiment.save()

        other_experiment = Experiment(name="experiment2")
        other_experiment.save()
        other_experiment.state = Experiment.DISABLED_STATE
        other_experiment.save()
        other_experiment.start_date = days[0].date()
        other_experiment.end_date = days[4].date()
        other_experiment.save()

        goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]

        # experiment starts on days[2]
        # other experiment starts on days[0]

        mocker = mox.Mox()
        finder = mocker.CreateMockAnything()
        calculator = mocker.CreateMockAnything()

        default_data = {
            Participant.TEST_GROUP: {
                "count": 110,
                "conversions": [23, 12, 9]
            },
            Participant.CONTROL_GROUP: {
                "count": 130,
                "conversions": [12, 47, 5]
            }
        }
        day_2_data = {
            Participant.TEST_GROUP: {
                "count": 12,
                "conversions": [0, 2, 3]
            },
            Participant.CONTROL_GROUP: {
                "count": 7,
                "conversions": [1, 0, 3]
            }
        }
        day_3_data = {
            Participant.TEST_GROUP: {
                "count": 5,
                "conversions": [1, 0, 3]
            },
            Participant.CONTROL_GROUP: {
                "count": 12,
                "conversions": [0, 0, 0]
            }
        }
        day_4_data = {
            Participant.TEST_GROUP: {
                "count": 0,
                "conversions": [0, 0, 0]
            },
            Participant.CONTROL_GROUP: {
                "count": 25,
                "conversions": [2, 3, 7]
            }
        }

        for day in days[2:7]:
            data = default_data
            if day == days[2]:
                data = day_2_data
            if day == days[3]:
                data = day_3_data
            if day == days[4]:
                data = day_4_data

            for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
                mock_participants = mocker.CreateMockAnything()
                finder(group, experiment,
                       day.date()).InAnyOrder().AndReturn(mock_participants)
                mock_participants.count().MultipleTimes().AndReturn(
                    data[group]["count"])
                for goal_type in goal_types:
                    calculator(goal_type, mock_participants,
                               day.date()).InAnyOrder().AndReturn(
                                   data[group]["conversions"][int(
                                       goal_type.name)])
                calculator(None, mock_participants,
                           day.date()).InAnyOrder().AndReturn(
                               sum(data[group]["conversions"]))
        mocker.ReplayAll()

        for d in days[2:7]:
            ConversionReportGenerator(
                calculator, finder).generate_daily_report_for_experiment(
                    experiment, d.date())

        results = DailyConversionReport.objects.filter(
            experiment=experiment).order_by('-date')

        mocker.VerifyAll()

        self.assertEquals(results.count(), 5)
        report_days = [d.date() for d in days[2:7]]
        for i in range(5):
            self.assertEquals(results[i].date, report_days[4 - i])

        # Day 2
        self.assertEquals(12, results[4].test_group_size)
        self.assertEquals(7, results[4].control_group_size)
        self.assertEquals(5, results[4].overall_test_conversion)
        self.assertEquals(4, results[4].overall_control_conversion)

        day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4], goal_type=goal_types[0])[0].test_conversion
        self.assertEquals(0, day_2_goal_4_test_conversion)

        day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4], goal_type=goal_types[2])[0].control_conversion
        self.assertEquals(3, day_2_goal_2_control_conversion)

        # Day 3
        self.assertEquals(5, results[3].test_group_size)
        # Day 4
        self.assertEquals(0, results[2].test_group_size)
        self.assertEquals(None, results[2].confidence)
        day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[2], goal_type=goal_types[0])[0].confidence
        self.assertEquals(None, day_4_goal_1_confidence)
        # Day 5
        day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[1], goal_type=goal_types[0])[0].confidence
        self.assertAlmostEqual(98.935467172597029,
                               day_5_goal_0_confidence,
                               places=6)
    def testExperimentGroupParticipantFinder(self):
        days = [
            datetime.combine(date.today() + timedelta(days=i), time(hour=12))
            for i in range(-7, 0)
        ]

        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = days[2].date()
        experiment.save()

        other_experiment = Experiment(name="experiment2")
        other_experiment.save()
        other_experiment.state = Experiment.DISABLED_STATE
        other_experiment.save()
        other_experiment.start_date = days[0].date()
        other_experiment.end_date = days[4].date()
        other_experiment.save()

        anonymous_visitors = [
            AnonymousVisitor.objects.create() for i in range(10)
        ]

        experiment_participant_groups = [
            [
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[0],
                    experiment=experiment,
                    enrollment_date=days[2],
                    group=Participant.TEST_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[1],
                    experiment=experiment,
                    enrollment_date=days[2],
                    group=Participant.CONTROL_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[3],
                    experiment=experiment,
                    enrollment_date=days[3],
                    group=Participant.TEST_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[4],
                    experiment=experiment,
                    enrollment_date=days[4],
                    group=Participant.CONTROL_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[6],
                    experiment=experiment,
                    enrollment_date=days[6],
                    group=Participant.TEST_GROUP)
            ],
            [
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[0],
                    experiment=other_experiment,
                    enrollment_date=days[0],
                    group=Participant.CONTROL_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[2],
                    experiment=other_experiment,
                    enrollment_date=days[0],
                    group=Participant.TEST_GROUP),
                self.create_participant(
                    anonymous_visitor=anonymous_visitors[5],
                    experiment=other_experiment,
                    enrollment_date=days[4],
                    group=Participant.TEST_GROUP)
            ]
        ]

        ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP,
                                                     experiment, days[2])
        ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
        self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
        self.assertEquals(1, len(ex1day2visitors))

        ex1day4test = find_experiment_group_participants(
            Participant.TEST_GROUP, experiment, days[4])
        ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
        self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
        self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
        self.assertEquals(2, len(ex1day4testvisitors))

        ex1day4control = find_experiment_group_participants(
            Participant.CONTROL_GROUP, experiment, days[4])
        ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
        self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
        self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
        self.assertEquals(2, len(ex1day4controlvisitors))

        ex2day5test = find_experiment_group_participants(
            Participant.TEST_GROUP, other_experiment, days[5])
        ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
        self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
        self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
        self.assertEquals(2, len(ex2day5testvisitors))
    def testGetConversionData(self):
        days = [
            datetime.combine(date.today() + timedelta(days=i), time(hour=12))
            for i in range(-7, 0)
        ]

        yesterday = date.today() - timedelta(days=1)
        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = yesterday
        experiment.save()

        goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]

        report = DailyConversionReport.objects.create(
            experiment=experiment,
            date=yesterday,
            overall_test_conversion=17,
            overall_control_conversion=12,
            test_group_size=139,
            control_group_size=142,
            confidence=87.3)

        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[0],
                                                     test_conversion=11,
                                                     control_conversion=0,
                                                     confidence=65.3)
        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[1],
                                                     test_conversion=0,
                                                     control_conversion=21,
                                                     confidence=None)
        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[2],
                                                     test_conversion=23,
                                                     control_conversion=21,
                                                     confidence=100)

        data = get_conversion_data(experiment, yesterday)

        self.assertEquals(data['date'], yesterday)
        self.assertTrue("totals" in data)
        self.assertTrue("goal_types" in data)
        self.assertEquals(4, len(data["goal_types"].keys()))

        for goal_type in goal_types[0:3]:
            self.assertTrue(goal_type.name in data["goal_types"])

            goal_type_data = data["goal_types"][goal_type.name]
            self.assertTrue("test_count" in goal_type_data)
            self.assertTrue("control_count" in goal_type_data)
            self.assertTrue("test_rate" in goal_type_data)
            self.assertTrue("control_rate" in goal_type_data)
            self.assertTrue("improvement" in goal_type_data)
            self.assertTrue("confidence" in goal_type_data)

        self.assertEquals(None, data["goal_types"][goal_types[3].name])

        self.assertEquals(139, data["test_group_size"])
        self.assertEquals(142, data["control_group_size"])

        totals = data["totals"]

        expected_test_rate = 17. / 139. * 100.
        expected_control_rate = 12. / 142. * 100.
        expected_improvement = (expected_test_rate - expected_control_rate
                                ) / expected_control_rate * 100.

        self.assertAlmostEquals(expected_test_rate, totals["test_rate"])
        self.assertAlmostEquals(expected_control_rate, totals["control_rate"])
        self.assertAlmostEquals(expected_improvement, totals["improvement"])
        self.assertAlmostEquals(87.3, totals["confidence"])
        self.assertEquals(17, totals["test_count"])
        self.assertEquals(12, totals["control_count"])

        self.assertEquals(
            0, data["goal_types"][goal_types[0].name]["control_rate"])
        self.assertAlmostEquals(
            11. / 139 * 100.,
            data["goal_types"][goal_types[0].name]["test_rate"])
        self.assertEquals(
            None, data["goal_types"][goal_types[0].name]["improvement"])
        self.assertAlmostEquals(
            65.3, data["goal_types"][goal_types[0].name]["confidence"])
        self.assertEquals(11,
                          data["goal_types"][goal_types[0].name]["test_count"])
        self.assertEquals(
            0, data["goal_types"][goal_types[0].name]["control_count"])

        self.assertAlmostEquals(
            21. / 142 * 100.,
            data["goal_types"][goal_types[1].name]["control_rate"])
        self.assertEquals(None,
                          data["goal_types"][goal_types[1].name]["confidence"])
        self.assertEquals(
            None, data["goal_types"][goal_types[1].name]["improvement"])

        self.assertAlmostEquals(
            (23. / 139 - 21. / 142) / (21. / 142) * 100.,
            data["goal_types"][goal_types[2].name]["improvement"])
Exemple #36
0
class TestExperimentViews(object):
    urls = 'experiments.tests.urls'

    def setUp(self):
        staff_user = User(username="******",
                          email="*****@*****.**",
                          is_staff=True)
        staff_user.save()
        staff_user.set_password("staff")
        staff_user.save()

        # self.assertTrue(self.client.login(username='******',
        #                                   password='******'))

        self.experiment1 = Experiment(name="experiment 1")
        self.experiment1.save()
        self.experiment1.state = Experiment.ENABLED_STATE
        self.experiment1.save()
        self.experiment1.start_date -= timedelta(days=5)
        self.experiment1.save()

        goal_types = [
            GoalType.objects.create(name='test_goal_%s' % i) for i in range(3)
        ]

        for i in range(1, 6):
            DailyEngagementReport.objects.create(date=days_ago(i),
                                                 experiment=self.experiment1,
                                                 control_score=3.2,
                                                 test_score=2.3,
                                                 control_group_size=3,
                                                 test_group_size=5,
                                                 confidence=93.2)
            conversion_report = DailyConversionReport.objects.create(
                date=days_ago(i),
                experiment=self.experiment1,
                overall_test_conversion=12,
                overall_control_conversion=9,
                test_group_size=39,
                control_group_size=27,
                confidence=87.4)
            for goal_type in goal_types:
                DailyConversionReportGoalData.objects.create(
                    report=conversion_report,
                    goal_type=goal_type,
                    test_conversion=11,
                    control_conversion=7,
                    confidence=45.3)

        self.experiment2 = Experiment(name="experiment 2")
        self.experiment2.save()

        self.experiment3 = Experiment(name="experiment 3")
        self.experiment3.save()
        self.experiment3.state = Experiment.ENABLED_STATE
        self.experiment3.save()
        self.experiment3.state = Experiment.PROMOTED_STATE
        self.experiment3.save()

    def testListExperimentsView(self):
        url = reverse('experiments.views.list_experiments')
        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)
        header, data = get_tables(response.content)[0]
        data.sort()
        self.assertEquals([u'Name', u'Status', u'Start Date', u'End Date'],
                          header)
        self.assertEquals([[
            u'experiment 1', u'Enabled',
            unicode(self.experiment1.start_date), u'None'
        ], [u'experiment 2', u'Disabled', u'None', u'None'],
                           [
                               u'experiment 3', u'Promoted',
                               unicode(self.experiment3.start_date),
                               unicode(self.experiment3.end_date)
                           ]], data)

    def testAnonymousUserCannotAccessReports(self):
        self.client.logout()
        url = reverse('experiments.views.experiment_details',
                      args=['experiment 1'])
        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)
        url = reverse('experiments.views.list_experiments')
        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)

    def testRegularUserCannotAccessReports(self):
        self.client.logout()

        regular_user = User(username="******", email="*****@*****.**")
        regular_user.save()
        regular_user.set_password("joe")
        regular_user.save()

        self.assertTrue(
            self.client.login(username='******', password='******'))

        url = reverse('experiments.views.experiment_details',
                      args=['experiment 1'])
        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)
        url = reverse('experiments.views.list_experiments')
        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)
        self.assertTrue('<title>Log in' in response.content)

    def test404IfExperimentDoesntExist(self):
        url = reverse('experiments.views.experiment_details',
                      args=['inexistant experiment'])
        response = self.client.get(url)
        self.assertEquals(response.status_code, 404)

    def testExperimentDetailsView(self):
        url = reverse('experiments.views.experiment_details',
                      args=['experiment 1'])
        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)
        tables = get_tables(response.content)
        experiment_properties = tables.pop(0)
        self.assertEquals([u'Property', u'Value'], experiment_properties[0])
        self.assertEquals(
            [[u'Experiment Name', u'experiment 1'], [u'Status', u'Enabled'],
             [u'Start Date',
              unicode(self.experiment1.start_date)], [u'End Date', u'None']],
            experiment_properties[1])
        conversion_summary = tables.pop(0)
        self.assertEquals([
            u'&nbsp;', u'Control Group', u'Test Group', u'Improvement',
            u'Confidence'
        ], conversion_summary[0])
        self.assertEquals(
            [[u'Participants', u'27', u'39', u'&nbsp;', u'&nbsp;'],
             [u'test_goal_0', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'],
             [u'test_goal_1', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'],
             [u'test_goal_2', u'7 (25.9 %)', u'11 (28.2 %)', u'8 %', u'45 %'],
             [u'Any', u'9 (33.3 %)', u'12 (30.8 %)', u'-7 %', u'87 %']],
            conversion_summary[1])
        engagement_summary = tables.pop(0)
        self.assertEquals([
            u'&nbsp;', u'Control Group', u'Test Group', u'Improvement',
            u'Confidence'
        ], engagement_summary[0])
        self.assertEquals(
            [[u'Participants', u'3', u'5', u'&nbsp;', u'&nbsp;'],
             [u'Engagement Score', u'3.2', u'2.3', u'-28 %', u'93 %']],
            engagement_summary[1])
        conversion_details = tables.pop(0)
        self.assertTrue(conversion_details[0])
        self.assertTrue(conversion_details[1])
        engagement_details = tables.pop(0)
        self.assertEquals([
            u'Report Date', u'Control Group Size', u'Control Group Score',
            u'Test Group Size', u'Test Group Score', u'Test Group Improvement',
            u'Confidence'
        ], engagement_details[0])
        self.assertEquals([
            unicode(days_ago(1)), u'3', u'3.2', u'5', u'2.3', u'-28 %', u'93 %'
        ], engagement_details[1][0])
        self.assertEquals([], tables)

    def testVerifyHuman(self):
        experiment = Experiment(name="experiment")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()

        other_experiment = Experiment(name="other_experiment")
        other_experiment.save()
        other_experiment.state = Experiment.ENABLED_STATE
        other_experiment.save()

        self.client = Client()

        original_participants_count = Participant.objects.all().count()
        original_anonymous_visitors_count = AnonymousVisitor.objects.all(
        ).count()
        experiment_url = reverse('experiments.tests.views.experiment_test',
                                 args=[experiment.name])
        response = self.client.get(experiment_url)
        self.assertEquals(response.status_code, 200)

        self.assertEquals(original_participants_count,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count,
                          AnonymousVisitor.objects.all().count())

        confirm_human_url = reverse('experiments.views.confirm_human')
        response = self.client.get(confirm_human_url)
        self.assertEquals(response.status_code, 204)
        self.assertEquals(0, len(response.content))
        self.assertEquals(original_participants_count + 1,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())

        # calling it again does nothing
        response = self.client.get(confirm_human_url)
        self.assertEquals(response.status_code, 204)
        self.assertEquals(0, len(response.content))
        self.assertEquals(original_participants_count + 1,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())

        other_experiment_url = reverse(
            'experiments.tests.views.experiment_test',
            args=[other_experiment.name])
        response = self.client.get(other_experiment_url)
        self.assertEquals(response.status_code, 200)

        # a new participant is immediately created for the new experiment
        self.assertEquals(original_participants_count + 2,
                          Participant.objects.all().count())
        # the existing anonymous visitor is reused
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())

        # the new experiment does not cause storage of a temporary enrollment
        response = self.client.get(confirm_human_url)
        self.assertEquals(response.status_code, 204)
        self.assertEquals(0, len(response.content))
        self.assertEquals(original_participants_count + 2,
                          Participant.objects.all().count())
        self.assertEquals(original_anonymous_visitors_count + 1,
                          AnonymousVisitor.objects.all().count())

    def testGroupSanity(self):
        experiment = Experiment(name="experiment")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment_url = reverse('experiments.tests.views.experiment_test',
                                 args=[experiment.name])
        for i in range(100):
            self.client = Client()
            response = self.client.get(experiment_url)
            self.assertEquals(response.status_code, 200)
            self.assertTrue(response.content.strip().lower() in ("test",
                                                                 "control"))
Exemple #37
0
    def testParticipants(self):
        user1 = User(username="******", email="*****@*****.**")
        user1.save()
        user2 = User(username="******", email="*****@*****.**")
        user2.save()
        user3 = User(username="******", email="*****@*****.**")
        user3.save()

        anonymous_visitor1 = AnonymousVisitor()
        anonymous_visitor1.save()
        anonymous_visitor2 = AnonymousVisitor()
        anonymous_visitor2.save()
        anonymous_visitor3 = AnonymousVisitor()
        anonymous_visitor3.save()

        experiment1 = Experiment(name="Experiment 1")
        experiment1.save()
        experiment2 = Experiment(name="Experiment 2")
        experiment2.save()

        participant11 = Participant(user=user1,
                                    experiment=experiment1,
                                    group=Participant.CONTROL_GROUP)
        participant11.save()
        participant12 = Participant(user=user1,
                                    experiment=experiment2,
                                    group=Participant.TEST_GROUP)
        participant12.save()
        participant21 = Participant(user=user2,
                                    experiment=experiment1,
                                    group=Participant.CONTROL_GROUP)
        participant21.save()
        participant22 = Participant(user=user2,
                                    experiment=experiment2,
                                    group=Participant.CONTROL_GROUP)
        participant22.save()

        self.assertRaises(
            Exception,
            lambda: Participant(user=user2,
                                experiment=experiment2,
                                group=Participant.TEST_GROUP).save())

        # create anonymous participants
        participant31 = Participant(anonymous_visitor=anonymous_visitor1,
                                    experiment=experiment1,
                                    group=Participant.CONTROL_GROUP)
        participant31.save()
        participant32 = Participant(anonymous_visitor=anonymous_visitor1,
                                    experiment=experiment2,
                                    group=Participant.TEST_GROUP)
        participant32.save()
        participant42 = Participant(anonymous_visitor=anonymous_visitor2,
                                    experiment=experiment2,
                                    group=Participant.CONTROL_GROUP)
        participant42.save()

        self.assertRaises(
            Exception,
            lambda: Participant(anonymous_visitor=anonymous_visitor1,
                                experiment=experiment1,
                                group=Participant.TEST_GROUP).save())

        self.assertRaises(
            Exception,
            lambda: Participant(experiment=experiment1,
                                group=Participant.TEST_GROUP).save())

        self.assertRaises(
            Exception,
            lambda: Participant(user=user3,
                                anonymous_visitor=anonymous_visitor3,
                                experiment=experiment1,
                                group=Participant.TEST_GROUP))
        self.assertRaises(
            Exception,
            lambda: Participant(user=user3, experiment=experiment1).save())

        self.assertNotEquals(participant11.enrollment_date, None)

        participant11.enrollment_date = None
        self.assertRaises(Exception, lambda: participant11.save())

        participant12.group = None
        self.assertRaises(Exception, lambda: participant12.save())
    def testAnonymousUser(self):
        experiment = Experiment(name="enabled")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        for i in range(100):
            user = TestUser()
            in_test = Experiment.test("enabled", user)
            anonymous_id = user.get_anonymous_id()
            self.assertNotEquals(None, anonymous_id)
            in_control = Experiment.control("enabled", user)
            self.assertEquals(user.get_anonymous_id(), anonymous_id)
            self.assertNotEquals(in_test, in_control)
            self.assertEquals(in_test, Experiment.test("enabled", user))
            self.assertEquals(user.get_anonymous_id(), anonymous_id)
            self.assertEquals(in_control, Experiment.control("enabled", user))
            self.assertEquals(user.get_anonymous_id(), anonymous_id)

            if in_test:
                test_user = user
            if in_control:
                control_user = user

        self.assertTrue(test_user and control_user)

        user = TestUser()
        experiment = Experiment(name="disabled")
        experiment.save()
        self.assertFalse(Experiment.test("disabled", user))
        self.assertEquals(None, user.get_anonymous_id())
Exemple #39
0
    def testParticipantEnrollment(self):
        experiment1 = Experiment(name="Experiment 1")
        experiment1.save()
        experiment1.state = Experiment.ENABLED_STATE
        experiment1.save()
        experiment2 = Experiment(name="Experiment 2")
        experiment2.save()
        experiment2.state = Experiment.ENABLED_STATE
        experiment2.save()

        num_control1 = 0
        num_test1 = 0
        num_control2 = 0
        num_test2 = 0

        for i in range(1000):
            username = "******" % i
            in_test1 = Experiment.test(experiment1.name,
                                       TestUser(username=username))
            self.assertEquals(
                in_test1, not Experiment.control(experiment1.name,
                                                 TestUser(username=username)))
            if in_test1:
                num_test1 += 1
            else:
                num_control1 += 1

            in_test2 = not Experiment.control(experiment2.name,
                                              TestUser(username=username))
            self.assertEquals(
                in_test2,
                Experiment.test(experiment2.name, TestUser(username=username)))

            if in_test2:
                num_test2 += 1
            else:
                num_control2 += 1

        self.assert_(num_control1 > 400)
        self.assert_(num_control2 > 400)
        self.assert_(num_test1 > 400)
        self.assert_(num_test2 > 400)
def pirates_v_zombies(request, *args, **kwargs):
    if Experiment.test("pirates_v_zombies", WebUser(request)):
        view = pirates
    else:
        view = zombies
    return view(request, *args, **kwargs)
Exemple #41
0
    def testExperimentStates(self):
        experiment1 = Experiment(name="test_experiment_1")
        experiment1.save()

        experiment1 = Experiment.objects.get(name="test_experiment_1")

        self.assertRaises(Exception,
                          lambda: Experiment(name="test_experiment_1").save())

        #test default values
        self.assertEquals(experiment1.state, Experiment.DISABLED_STATE)
        self.assertEquals(experiment1.start_date, None)
        self.assertEquals(experiment1.end_date, None)

        #enable the experiment
        experiment1.state = Experiment.ENABLED_STATE
        experiment1.save()

        self.assertNotEquals(experiment1.start_date, None)
        self.assertEquals(experiment1.end_date, None)

        #disable the experiement
        old_start_date = experiment1.start_date
        experiment1.state = Experiment.DISABLED_STATE
        experiment1.save()
        self.assertEquals(experiment1.start_date, old_start_date)
        self.assertNotEquals(experiment1.end_date, None)

        #enable the experiment
        old_end_date = experiment1.end_date
        experiment1.state = Experiment.ENABLED_STATE
        experiment1.save()

        self.assertEquals(experiment1.start_date, old_start_date)
        self.assertEquals(experiment1.end_date, old_end_date)

        #promote the experiment
        experiment1.state = Experiment.PROMOTED_STATE
        experiment1.save()

        self.assertEquals(experiment1.start_date, old_start_date)
        self.assertEquals(experiment1.end_date, old_end_date)

        experiment2 = Experiment(name="Experiment 2")
        experiment2.save()
        experiment2.state = Experiment.ENABLED_STATE
        experiment2.save()
        self.assertEquals(experiment2.start_date, experiment1.start_date)
        experiment2.state = Experiment.PROMOTED_STATE
        experiment2.save()

        self.assertNotEquals(experiment2.start_date, None)
        self.assertNotEquals(experiment2.end_date, None)
class TestDailyReports(TestCase):
    def setUp(self):
        self.experiment = Experiment(name="test_experiment")
        self.experiment.save()
        self.experiment.state = Experiment.ENABLED_STATE
        self.experiment.save()
        self.experiment.start_date = (self.experiment.start_date -
                                      timedelta(days=5))
        self.experiment.save()
        
        anonymous_visitor = AnonymousVisitor()
        anonymous_visitor.save()
        anonymous_participant = Participant(anonymous_visitor=anonymous_visitor,
                                            experiment=self.experiment,
                                            group=Participant.TEST_GROUP)
        anonymous_participant.save()
        anonymous_participant.enrollment_date = self.experiment.start_date
        anonymous_participant.save()
        
        self.other_experiment = Experiment(name="test_experiment2")
        self.other_experiment.save()
        self.other_experiment.state = Experiment.DISABLED_STATE
        self.other_experiment.save()
        self.other_experiment.start_date = (date.today() -
                                      timedelta(days=7))
        self.other_experiment.end_date = (date.today() -
                                      timedelta(days=3))
        self.other_experiment.save()
    
    def testDailyEngagementReport(self):
        users_test = []
        users_control = []
        
        num_control1 = 0
        num_test1 = 0
        num_control2 = 0
        num_test2 = 0
        
        #create users
        for i in range(5):
            users_control.append(create_user_in_group(self.experiment, i,
                                        Participant.CONTROL_GROUP,
                                        date.today() - timedelta(days=i)))
            users_test.append(create_user_in_group(self.experiment, i,
                                        Participant.TEST_GROUP,
                                        date.today() - timedelta(days=i)))
        
        # users_<test|control>[0] were enrolled today, [1] 1 day ago, etc.
        
        report_date = date.today() - timedelta(days=1)
        expected_engagement_score_calls = {
            (users_test[1], date.today() - timedelta(days=1), report_date): 3.2,
            (users_test[2], date.today() - timedelta(days=2), report_date): 2.5,
            (users_test[3], date.today() - timedelta(days=3), report_date): 4.1,
            (users_test[4], date.today() - timedelta(days=4), report_date): 0,
            (users_control[1], date.today() - timedelta(days=1), report_date): 0,
            (users_control[2], date.today() - timedelta(days=2), report_date): 0,
            (users_control[3], date.today() - timedelta(days=3), report_date): 0,
            (users_control[4], date.today() - timedelta(days=4), report_date): 0}
        
        test_case = self
        
        class EngagementScoreCalculatorStub(object):
            def calculate_user_engagement_score(self, user,
                                              start_date, end_date):
                test_case.assertNotEqual(user, None)
                test_case.assertTrue((user, start_date, end_date) in expected_engagement_score_calls)
                return expected_engagement_score_calls[(user,
                                                     start_date, end_date)]
        
        (EngagementReportGenerator(EngagementScoreCalculatorStub()).
           generate_daily_report_for_experiment(self.experiment, report_date))

        
        experiment_report = DailyEngagementReport.objects.get(
                                experiment=self.experiment, date=report_date)
        self.assertAlmostEqual((3.2 + 2.5 + 4.1 + 0)/4.0,
                                experiment_report.test_score)
        self.assertAlmostEqual(0.0, experiment_report.control_score)
        self.assertEqual(4, experiment_report.test_group_size)
        self.assertEqual(4, experiment_report.control_group_size)
        self.assertAlmostEqual(96.819293337188498, experiment_report.confidence)
    
    def testZeroParticipantExperiment(self):
        mocker = mox.Mox()
        engagement_calculator = mocker.CreateMockAnything()
        mocker.ReplayAll()
        
        report_date = date.today()
        EngagementReportGenerator(engagement_score_calculator=engagement_calculator).generate_daily_report_for_experiment(
            self.other_experiment, report_date)
        
        experiment_report = DailyEngagementReport.objects.get(
            experiment=self.other_experiment, date=report_date)
        
        mocker.VerifyAll()
        
        self.assertEqual(None, experiment_report.test_score)
        self.assertEqual(None, experiment_report.control_score)
        self.assertEqual(0, experiment_report.test_group_size)
        self.assertEqual(0, experiment_report.control_group_size)
    
    def testGenerateAllDailyEngagementReports(self):
        class DummyEngagementCalculator(object):
            def calculate_user_engagement_score(self, user, start_date, end_date):
                return 7
        engagement_report_generator = EngagementReportGenerator(engagement_score_calculator=DummyEngagementCalculator())
        engagement_report_generator.generate_daily_report_for_experiment(
                            self.experiment, date.today() - timedelta(days=2))
        engagement_report_generator.generate_daily_report_for_experiment(
                            self.experiment, date.today() - timedelta(days=3))
        engagement_report_generator.generate_all_daily_reports()
        DailyEngagementReport.objects.get(experiment=self.experiment,
                            date=date.today() - timedelta(days=1))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                            date=date.today() - timedelta(days=2))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                            date=date.today() - timedelta(days=3))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                            date=date.today() - timedelta(days=4))
        DailyEngagementReport.objects.get(experiment=self.experiment,
                            date=date.today() - timedelta(days=5))
        self.assertEqual(5, DailyEngagementReport.objects.filter(
                                        experiment=self.experiment).count())
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                          date=date.today() - timedelta(days=3))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                          date=date.today() - timedelta(days=4))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                          date=date.today() - timedelta(days=5))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                          date=date.today() - timedelta(days=6))
        DailyEngagementReport.objects.get(experiment=self.other_experiment,
                          date=date.today() - timedelta(days=7))
        
        self.assertEqual(5, DailyEngagementReport.objects.filter(
                                    experiment=self.other_experiment).count())
    
    def create_goal_record(self, record_datetime, anonymous_visitor, goal_type):
        record = GoalRecord.objects.create(anonymous_visitor=anonymous_visitor,
                                           goal_type=goal_type)
        record.created = record_datetime
        record.save()
    
    def create_participant(self, anonymous_visitor, experiment, enrollment_date, group):
        participant = Participant.objects.create(anonymous_visitor=anonymous_visitor,
                                                 experiment=experiment,
                                                 group=group)
        participant.enrollment_date=enrollment_date
        participant.save()
        return participant
    
    def testParticipantConversionCalculator(self):
        goal_types = [GoalType.objects.create(name=str(i)) for i in range(3)]
        anonymous_visitor = AnonymousVisitor.objects.create()
        participant = self.create_participant(
            anonymous_visitor=anonymous_visitor,
            experiment=self.experiment,
            enrollment_date=self.experiment.start_date + timedelta(days=2),
            group=Participant.TEST_GROUP)
        
        days = [datetime.combine(self.experiment.start_date + timedelta(days=i), time(hour=12))
                for i in range(5)]
        
        nb_goal_records = GoalRecord.objects.all().count()
        
        self.create_goal_record(days[0], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[0], anonymous_visitor, goal_types[1])
        self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[2], anonymous_visitor, goal_types[1])
        self.create_goal_record(days[3], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[4], anonymous_visitor, goal_types[0])
        self.create_goal_record(days[4], anonymous_visitor, goal_types[0])
        
        self.assertEqual(nb_goal_records + 8, GoalRecord.objects.all().count())
        
        # wasn't enrolled yet!
        self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[0]), 0)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[0]), 0)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[0]), 0)
        self.assertEqual(calculate_participant_conversion(participant, None, days[0]), 0)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[1]), 0)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[1]), 0)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[1]), 0)
        self.assertEqual(calculate_participant_conversion(participant, None, days[1]), 0)
        
        # now enrolled
        self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[2]), 0)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[2]), 1)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[2]), 0)
        # "any" is one
        self.assertEqual(calculate_participant_conversion(participant, None, days[2]), 1)
        
        self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[3]), 1)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[3]), 1)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[3]), 0)
        # "any" is one, even though two different goals were achieved
        self.assertEqual(calculate_participant_conversion(participant, None, days[3]), 1)
        
        # there were three conversions on this day for goal 0, but we only count the first!
        self.assertEqual(calculate_participant_conversion(participant, goal_types[0], days[4]), 1)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[1], days[4]), 1)
        self.assertEqual(calculate_participant_conversion(participant, goal_types[2], days[4]), 0)
        self.assertEqual(calculate_participant_conversion(participant, None, days[4]), 1)
    
    def testGoalTypeConversionCalculator(self):
        mocker = mox.Mox()
        participants = [mocker.CreateMockAnything(),
                        mocker.CreateMockAnything(),
                        mocker.CreateMockAnything()]
        goal_type = mocker.CreateMockAnything()
        report_date = mocker.CreateMockAnything()
        participant_conversion_calculator = mocker.CreateMockAnything()
        
        participant_conversion_calculator(
            participants[0], goal_type, report_date).InAnyOrder().AndReturn(1)
        participant_conversion_calculator(
            participants[1], goal_type, report_date).InAnyOrder().AndReturn(0)
        participant_conversion_calculator(
            participants[2], goal_type, report_date).InAnyOrder().AndReturn(1)
        
        mocker.ReplayAll()
        
        self.assertEqual(2, calculate_goal_type_conversion(
                goal_type, participants, report_date, participant_conversion_calculator))
        
        mocker.VerifyAll()
    
    def testExperimentGroupParticipantFinder(self):
        days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
                for i in range(-7, 0)]
        
        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = days[2].date()
        experiment.save()
        
        other_experiment = Experiment(name="experiment2")
        other_experiment.save()
        other_experiment.state = Experiment.DISABLED_STATE
        other_experiment.save()
        other_experiment.start_date = days[0].date()
        other_experiment.end_date = days[4].date()
        other_experiment.save()
        
        anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)]
        
        experiment_participant_groups = [
            [
                self.create_participant(anonymous_visitor=anonymous_visitors[0],
                                        experiment=experiment,
                                        enrollment_date=days[2],
                                        group=Participant.TEST_GROUP),
                self.create_participant(anonymous_visitor=anonymous_visitors[1],
                                        experiment=experiment,
                                        enrollment_date=days[2],
                                        group=Participant.CONTROL_GROUP),
                self.create_participant(anonymous_visitor=anonymous_visitors[3],
                                        experiment=experiment,
                                        enrollment_date=days[3],
                                        group=Participant.TEST_GROUP),
                self.create_participant(anonymous_visitor=anonymous_visitors[4],
                                        experiment=experiment,
                                        enrollment_date=days[4],
                                        group=Participant.CONTROL_GROUP),
                self.create_participant(anonymous_visitor=anonymous_visitors[6],
                                        experiment=experiment,
                                        enrollment_date=days[6],
                                        group=Participant.TEST_GROUP)
                ],
            [
                self.create_participant(anonymous_visitor=anonymous_visitors[0],
                                        experiment=other_experiment,
                                        enrollment_date=days[0],
                                        group=Participant.CONTROL_GROUP),
                self.create_participant(anonymous_visitor=anonymous_visitors[2],
                                        experiment=other_experiment,
                                        enrollment_date=days[0],
                                        group=Participant.TEST_GROUP),
                self.create_participant(anonymous_visitor=anonymous_visitors[5],
                                        experiment=other_experiment,
                                        enrollment_date=days[4],
                                        group=Participant.TEST_GROUP)
                ]
            ]
        
        ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2])
        ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
        self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
        self.assertEqual(1, len(ex1day2visitors))
        
        ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4])
        ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
        self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
        self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
        self.assertEqual(2, len(ex1day4testvisitors))
        
        ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4])
        ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
        self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
        self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
        self.assertEqual(2, len(ex1day4controlvisitors))
        
        ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5])
        ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
        self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
        self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
        self.assertEqual(2, len(ex2day5testvisitors))
    
    def testGetConversionData(self):
        days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
                for i in range(-7, 0)]
        
        yesterday = date.today() - timedelta(days=1)
        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = yesterday
        experiment.save()
        
        goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
        
        report = DailyConversionReport.objects.create(experiment=experiment,
                                             date=yesterday,
                                             overall_test_conversion=17,
                                             overall_control_conversion=12,
                                             test_group_size=139,
                                             control_group_size=142,
                                             confidence=87.3)
        
        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[0],
                                                     test_conversion=11,
                                                     control_conversion=0,
                                                     confidence=65.3)
        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[1],
                                                     test_conversion=0,
                                                     control_conversion=21,
                                                     confidence=None)
        DailyConversionReportGoalData.objects.create(report=report,
                                                     goal_type=goal_types[2],
                                                     test_conversion=23,
                                                     control_conversion=21,
                                                     confidence=100)
        
        data = get_conversion_data(experiment, yesterday)
        
        self.assertEqual(data['date'], yesterday)
        self.assertTrue("totals" in data)
        self.assertTrue("goal_types" in data)
        self.assertEqual(4, len(data["goal_types"].keys()))
        
        for goal_type in goal_types[0:3]:
            self.assertTrue(goal_type.name in data["goal_types"])
            
            goal_type_data = data["goal_types"][goal_type.name]
            self.assertTrue("test_count" in goal_type_data)
            self.assertTrue("control_count" in goal_type_data)
            self.assertTrue("test_rate" in goal_type_data)
            self.assertTrue("control_rate" in goal_type_data)
            self.assertTrue("improvement" in goal_type_data)
            self.assertTrue("confidence" in goal_type_data)
        
        self.assertEqual(None, data["goal_types"][goal_types[3].name])
        
        self.assertEqual(139, data["test_group_size"])
        self.assertEqual(142, data["control_group_size"])
        
        totals = data["totals"]
        
        expected_test_rate = 17. / 139. * 100.
        expected_control_rate = 12. / 142. * 100.
        expected_improvement = (expected_test_rate - expected_control_rate) / expected_control_rate * 100.
        
        self.assertAlmostEqual(expected_test_rate, totals["test_rate"])
        self.assertAlmostEqual(expected_control_rate, totals["control_rate"])
        self.assertAlmostEqual(expected_improvement, totals["improvement"])
        self.assertAlmostEqual(87.3, totals["confidence"])
        self.assertEqual(17, totals["test_count"])
        self.assertEqual(12, totals["control_count"])
        
        self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_rate"])
        self.assertAlmostEqual(11./139*100., data["goal_types"][goal_types[0].name]["test_rate"])
        self.assertEqual(None, data["goal_types"][goal_types[0].name]["improvement"])
        self.assertAlmostEqual(65.3, data["goal_types"][goal_types[0].name]["confidence"])
        self.assertEqual(11, data["goal_types"][goal_types[0].name]["test_count"])
        self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_count"])
        
        self.assertAlmostEqual(21./142*100., data["goal_types"][goal_types[1].name]["control_rate"])
        self.assertEqual(None, data["goal_types"][goal_types[1].name]["confidence"])
        self.assertEqual(None, data["goal_types"][goal_types[1].name]["improvement"])
        
        self.assertAlmostEqual((23./139-21./142)/(21./142)*100.,
                                data["goal_types"][goal_types[2].name]["improvement"])


#TODO test with zero participants and check rate == None

#TODO sometimes confidence cannot be calculated and must return None. Add a test to verify this.
    
    def testConversionReportGenerator(self):
        days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
                for i in range(-7, 0)]
        
        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = days[2].date()
        experiment.save()
        
        other_experiment = Experiment(name="experiment2")
        other_experiment.save()
        other_experiment.state = Experiment.DISABLED_STATE
        other_experiment.save()
        other_experiment.start_date = days[0].date()
        other_experiment.end_date = days[4].date()
        other_experiment.save()

        
        goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
        
        # experiment starts on days[2]
        # other experiment starts on days[0]
        
        mocker = mox.Mox()
        finder = mocker.CreateMockAnything()
        calculator = mocker.CreateMockAnything()
        
        default_data = {
            Participant.TEST_GROUP: {
                "count": 110,
                "conversions": [23, 12, 9]
            },
            Participant.CONTROL_GROUP: {
                "count": 130,
                "conversions": [12, 47, 5]
                }
            }
        day_2_data = {
            Participant.TEST_GROUP: {
                "count": 12,
                "conversions": [0, 2, 3]
                },
            Participant.CONTROL_GROUP: {
                "count": 7,
                "conversions": [1, 0, 3]
                }
            }
        day_3_data = {
            Participant.TEST_GROUP: {
                "count": 5,
                "conversions": [1, 0, 3]
                },
            Participant.CONTROL_GROUP: {
                "count": 12,
                "conversions": [0, 0, 0]
                }
            }
        day_4_data = {
            Participant.TEST_GROUP: {
                "count": 0,
                "conversions": [0, 0, 0]
                },
            Participant.CONTROL_GROUP: {
                "count": 25,
                "conversions": [2, 3, 7]
                }
            }
        
        for day in days[2:7]:
            data = default_data
            if day == days[2]:
                data = day_2_data
            if day == days[3]:
                data = day_3_data
            if day == days[4]:
                data = day_4_data
            
            for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
                mock_participants = mocker.CreateMockAnything()
                finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants)
                mock_participants.count().MultipleTimes().AndReturn(data[group]["count"])
                for goal_type in goal_types:
                    calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)])
                calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"]))
        mocker.ReplayAll()
        
        for d in days[2:7]:
            ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment(
                experiment, d.date())
        
        results = DailyConversionReport.objects.filter(
            experiment=experiment).order_by('-date')
        
        mocker.VerifyAll()
        
        self.assertEqual(results.count(), 5)
        report_days = [ d.date() for d in days[2:7]]
        for i in range(5):
            self.assertEqual(results[i].date, report_days[4-i])
        
        # Day 2
        self.assertEqual(12, results[4].test_group_size)
        self.assertEqual(7, results[4].control_group_size)
        self.assertEqual(5, results[4].overall_test_conversion)
        self.assertEqual(4, results[4].overall_control_conversion)
        
        day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4],
            goal_type=goal_types[0])[0].test_conversion
        self.assertEqual(0, day_2_goal_4_test_conversion)
        
        day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4],
            goal_type=goal_types[2])[0].control_conversion
        self.assertEqual(3, day_2_goal_2_control_conversion)
        
        # Day 3
        self.assertEqual(5, results[3].test_group_size)
        # Day 4
        self.assertEqual(0, results[2].test_group_size)
        self.assertEqual(None, results[2].confidence)
        day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[2],
            goal_type=goal_types[0])[0].confidence
        self.assertEqual(None, day_4_goal_1_confidence)
        # Day 5
        day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[1],
            goal_type=goal_types[0])[0].confidence
        self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
 def testVerifyHuman(self):
     experiment = Experiment(name="experiment")
     experiment.save()
     experiment.state = Experiment.ENABLED_STATE
     experiment.save()
     
     other_experiment = Experiment(name="other_experiment")
     other_experiment.save()
     other_experiment.state = Experiment.ENABLED_STATE
     other_experiment.save()
     
     self.client = Client()
     
     original_participants_count = Participant.objects.all().count()
     original_anonymous_visitors_count = AnonymousVisitor.objects.all().count()
     experiment_url = reverse('experiments.tests.views.experiment_test',
                              args=[experiment.name])
     response = self.client.get(experiment_url)
     self.assertEqual(response.status_code, 200)
     
     self.assertEqual(original_participants_count,
                       Participant.objects.all().count())
     self.assertEqual(original_anonymous_visitors_count,
                       AnonymousVisitor.objects.all().count())
     
     confirm_human_url = reverse('experiments.views.confirm_human')
     response = self.client.get(confirm_human_url)
     self.assertEqual(response.status_code, 204)
     self.assertEqual(0, len(response.content))
     self.assertEqual(original_participants_count+1,
                       Participant.objects.all().count())
     self.assertEqual(original_anonymous_visitors_count+1,
                       AnonymousVisitor.objects.all().count())
     
     # calling it again does nothing
     response = self.client.get(confirm_human_url)
     self.assertEqual(response.status_code, 204)
     self.assertEqual(0, len(response.content))
     self.assertEqual(original_participants_count+1,
                       Participant.objects.all().count())
     self.assertEqual(original_anonymous_visitors_count+1,
                       AnonymousVisitor.objects.all().count())
     
     other_experiment_url = reverse(
         'experiments.tests.views.experiment_test',
         args=[other_experiment.name])
     response = self.client.get(other_experiment_url)
     self.assertEqual(response.status_code, 200)
     
     # a new participant is immediately created for the new experiment
     self.assertEqual(original_participants_count + 2,
                       Participant.objects.all().count())
     # the existing anonymous visitor is reused
     self.assertEqual(original_anonymous_visitors_count + 1,
                       AnonymousVisitor.objects.all().count())
     
     # the new experiment does not cause storage of a temporary enrollment
     response = self.client.get(confirm_human_url)
     self.assertEqual(response.status_code, 204)
     self.assertEqual(0, len(response.content))
     self.assertEqual(original_participants_count+2,
                       Participant.objects.all().count())
     self.assertEqual(original_anonymous_visitors_count+1,
                       AnonymousVisitor.objects.all().count())
 def testExperimentGroupParticipantFinder(self):
     days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
             for i in range(-7, 0)]
     
     experiment = Experiment(name="experiment1")
     experiment.save()
     experiment.state = Experiment.ENABLED_STATE
     experiment.save()
     experiment.start_date = days[2].date()
     experiment.save()
     
     other_experiment = Experiment(name="experiment2")
     other_experiment.save()
     other_experiment.state = Experiment.DISABLED_STATE
     other_experiment.save()
     other_experiment.start_date = days[0].date()
     other_experiment.end_date = days[4].date()
     other_experiment.save()
     
     anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)]
     
     experiment_participant_groups = [
         [
             self.create_participant(anonymous_visitor=anonymous_visitors[0],
                                     experiment=experiment,
                                     enrollment_date=days[2],
                                     group=Participant.TEST_GROUP),
             self.create_participant(anonymous_visitor=anonymous_visitors[1],
                                     experiment=experiment,
                                     enrollment_date=days[2],
                                     group=Participant.CONTROL_GROUP),
             self.create_participant(anonymous_visitor=anonymous_visitors[3],
                                     experiment=experiment,
                                     enrollment_date=days[3],
                                     group=Participant.TEST_GROUP),
             self.create_participant(anonymous_visitor=anonymous_visitors[4],
                                     experiment=experiment,
                                     enrollment_date=days[4],
                                     group=Participant.CONTROL_GROUP),
             self.create_participant(anonymous_visitor=anonymous_visitors[6],
                                     experiment=experiment,
                                     enrollment_date=days[6],
                                     group=Participant.TEST_GROUP)
             ],
         [
             self.create_participant(anonymous_visitor=anonymous_visitors[0],
                                     experiment=other_experiment,
                                     enrollment_date=days[0],
                                     group=Participant.CONTROL_GROUP),
             self.create_participant(anonymous_visitor=anonymous_visitors[2],
                                     experiment=other_experiment,
                                     enrollment_date=days[0],
                                     group=Participant.TEST_GROUP),
             self.create_participant(anonymous_visitor=anonymous_visitors[5],
                                     experiment=other_experiment,
                                     enrollment_date=days[4],
                                     group=Participant.TEST_GROUP)
             ]
         ]
     
     ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2])
     ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
     self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
     self.assertEqual(1, len(ex1day2visitors))
     
     ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4])
     ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
     self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
     self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
     self.assertEqual(2, len(ex1day4testvisitors))
     
     ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4])
     ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
     self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
     self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
     self.assertEqual(2, len(ex1day4controlvisitors))
     
     ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5])
     ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
     self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
     self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
     self.assertEqual(2, len(ex2day5testvisitors))
 def testExperimentStates(self):
     experiment1 = Experiment(name="test_experiment_1")
     experiment1.save()
     
     experiment1 = Experiment.objects.get(name="test_experiment_1")
     
     self.assertRaises(Exception,
                       lambda: Experiment(name="test_experiment_1").save())
     
     #test default values
     self.assertEquals(experiment1.state, Experiment.DISABLED_STATE)
     self.assertEquals(experiment1.start_date, None)
     self.assertEquals(experiment1.end_date, None)
     
     #enable the experiment
     experiment1.state = Experiment.ENABLED_STATE
     experiment1.save()
     
     self.assertNotEquals(experiment1.start_date, None)
     self.assertEquals(experiment1.end_date, None)
     
     #disable the experiement
     old_start_date = experiment1.start_date
     experiment1.state = Experiment.DISABLED_STATE
     experiment1.save()
     self.assertEquals(experiment1.start_date, old_start_date)
     self.assertNotEquals(experiment1.end_date, None)
     
     #enable the experiment
     old_end_date = experiment1.end_date
     experiment1.state = Experiment.ENABLED_STATE
     experiment1.save()
     
     self.assertEquals(experiment1.start_date, old_start_date)
     self.assertEquals(experiment1.end_date, old_end_date)
     
     #promote the experiment
     experiment1.state = Experiment.PROMOTED_STATE
     experiment1.save()
     
     self.assertEquals(experiment1.start_date, old_start_date)
     self.assertEquals(experiment1.end_date, old_end_date)
     
     experiment2 = Experiment(name= "Experiment 2")
     experiment2.save()
     experiment2.state = Experiment.ENABLED_STATE
     experiment2.save()
     self.assertEquals(experiment2.start_date, experiment1.start_date)
     experiment2.state = Experiment.PROMOTED_STATE
     experiment2.save()
     
     self.assertNotEquals(experiment2.start_date, None)
     self.assertNotEquals(experiment2.end_date, None)
    def testConversionReportGenerator(self):
        days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
                for i in range(-7, 0)]
        
        experiment = Experiment(name="experiment1")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        experiment.start_date = days[2].date()
        experiment.save()
        
        other_experiment = Experiment(name="experiment2")
        other_experiment.save()
        other_experiment.state = Experiment.DISABLED_STATE
        other_experiment.save()
        other_experiment.start_date = days[0].date()
        other_experiment.end_date = days[4].date()
        other_experiment.save()

        
        goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
        
        # experiment starts on days[2]
        # other experiment starts on days[0]
        
        mocker = mox.Mox()
        finder = mocker.CreateMockAnything()
        calculator = mocker.CreateMockAnything()
        
        default_data = {
            Participant.TEST_GROUP: {
                "count": 110,
                "conversions": [23, 12, 9]
            },
            Participant.CONTROL_GROUP: {
                "count": 130,
                "conversions": [12, 47, 5]
                }
            }
        day_2_data = {
            Participant.TEST_GROUP: {
                "count": 12,
                "conversions": [0, 2, 3]
                },
            Participant.CONTROL_GROUP: {
                "count": 7,
                "conversions": [1, 0, 3]
                }
            }
        day_3_data = {
            Participant.TEST_GROUP: {
                "count": 5,
                "conversions": [1, 0, 3]
                },
            Participant.CONTROL_GROUP: {
                "count": 12,
                "conversions": [0, 0, 0]
                }
            }
        day_4_data = {
            Participant.TEST_GROUP: {
                "count": 0,
                "conversions": [0, 0, 0]
                },
            Participant.CONTROL_GROUP: {
                "count": 25,
                "conversions": [2, 3, 7]
                }
            }
        
        for day in days[2:7]:
            data = default_data
            if day == days[2]:
                data = day_2_data
            if day == days[3]:
                data = day_3_data
            if day == days[4]:
                data = day_4_data
            
            for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
                mock_participants = mocker.CreateMockAnything()
                finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants)
                mock_participants.count().MultipleTimes().AndReturn(data[group]["count"])
                for goal_type in goal_types:
                    calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)])
                calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"]))
        mocker.ReplayAll()
        
        for d in days[2:7]:
            ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment(
                experiment, d.date())
        
        results = DailyConversionReport.objects.filter(
            experiment=experiment).order_by('-date')
        
        mocker.VerifyAll()
        
        self.assertEqual(results.count(), 5)
        report_days = [ d.date() for d in days[2:7]]
        for i in range(5):
            self.assertEqual(results[i].date, report_days[4-i])
        
        # Day 2
        self.assertEqual(12, results[4].test_group_size)
        self.assertEqual(7, results[4].control_group_size)
        self.assertEqual(5, results[4].overall_test_conversion)
        self.assertEqual(4, results[4].overall_control_conversion)
        
        day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4],
            goal_type=goal_types[0])[0].test_conversion
        self.assertEqual(0, day_2_goal_4_test_conversion)
        
        day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
            report=results[4],
            goal_type=goal_types[2])[0].control_conversion
        self.assertEqual(3, day_2_goal_2_control_conversion)
        
        # Day 3
        self.assertEqual(5, results[3].test_group_size)
        # Day 4
        self.assertEqual(0, results[2].test_group_size)
        self.assertEqual(None, results[2].confidence)
        day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[2],
            goal_type=goal_types[0])[0].confidence
        self.assertEqual(None, day_4_goal_1_confidence)
        # Day 5
        day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
            report=results[1],
            goal_type=goal_types[0])[0].confidence
        self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
    def testAnonymousUser(self):
        experiment = Experiment(name="enabled")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        for i in range(100):
            user = TestUser()
            in_test = Experiment.test("enabled", user)
            anonymous_id = user.get_anonymous_id()
            self.assertNotEqual(None, anonymous_id)
            in_control = Experiment.control("enabled", user)
            self.assertEqual(user.get_anonymous_id(), anonymous_id)
            self.assertNotEqual(in_test, in_control)
            self.assertEqual(in_test, Experiment.test("enabled", user))
            self.assertEqual(user.get_anonymous_id(), anonymous_id)
            self.assertEqual(in_control, Experiment.control("enabled", user))
            self.assertEqual(user.get_anonymous_id(), anonymous_id)

            if in_test:
                test_user = user
            if in_control:
                control_user = user

        self.assertTrue(test_user and control_user)

        user = TestUser()
        experiment = Experiment(name="disabled")
        experiment.save()
        self.assertFalse(Experiment.test("disabled", user))
        self.assertEqual(None, user.get_anonymous_id())
    def testEnabledPromotedAndDisabledExperiment(self):
        # enabled test, new user (prove we get both results)
        experiment = Experiment(name="enabled")
        experiment.save()
        experiment.state = Experiment.ENABLED_STATE
        experiment.save()
        test_user = None
        control_user = None
        for i in range(100):
            user = TestUser(username="******" % i)
            in_test = Experiment.test("enabled", user)
            in_control = Experiment.control("enabled", user)
            self.assertNotEqual(in_test, in_control)
            self.assertEqual(in_test, Experiment.test("enabled", user))
            self.assertEqual(in_control, Experiment.control("enabled", user))
            if in_test:
                test_user = user
            if in_control:
                control_user = user

        self.assertNotEqual(None, test_user)
        self.assertNotEqual(None, control_user)
        self.assertNotEqual(test_user, control_user)

        # promoted test with an enrolled user
        # (prove we get both results, then that all users are in test)
        experiment.state = Experiment.PROMOTED_STATE
        experiment.save()

        self.assertFalse(Experiment.control("enabled", control_user))
        self.assertFalse(Experiment.control("enabled", test_user))
        self.assertTrue(Experiment.test("enabled", control_user))
        self.assertTrue(Experiment.test("enabled", test_user))

        # disabled test with an enrolled user
        # (prove we get both results, then that all users are in control)
        experiment.state = Experiment.DISABLED_STATE
        experiment.save()

        self.assertTrue(Experiment.control("enabled", control_user))
        self.assertTrue(Experiment.control("enabled", test_user))
        self.assertFalse(Experiment.test("enabled", control_user))
        self.assertFalse(Experiment.test("enabled", test_user))
Exemple #49
0
class ExperimentTagsTest(TestCase):
    urls = 'django_lean.experiments.tests.urls'
    
    def setUp(self):
        self.experiment = Experiment(name="test_experiment")
        self.experiment.save()
        self.experiment.state = Experiment.ENABLED_STATE
        self.experiment.save()
        
        self.other_experiment = Experiment(name="other_test_experiment")
        self.other_experiment.save()
        self.other_experiment.state = Experiment.ENABLED_STATE
        self.other_experiment.save()
        self.mox = mox.Mox()
    
    def testIntegrationWithAnonymousVisitor(self):
        self.doTestIntegration(
            url=reverse('django_lean.experiments.tests.views.experiment_test',
                        args=[self.experiment.name]),
            client_factory=lambda i: Client())
        self.doTestIntegration(
            url=reverse('django_lean.experiments.tests.views.clientsideexperiment_test',
                        args=[self.other_experiment.name]),
            client_factory=lambda i: Client())
    
    def testIntegrationWithRegisteredUser(self):
        def create_registered_user_client(i):
            user = User(username="******" % i, email="*****@*****.**" % i)
            user.set_password("password")
            user.save()
            
            client = Client()
            
            if not client.login(username=user.username, password="******"):
                raise Exception("login failure")
            return client
        self.doTestIntegration(
            url=reverse('django_lean.experiments.tests.views.experiment_test',
                        args=[self.experiment.name]),
            client_factory=create_registered_user_client)
        
        # we wrap our factory to ensure that the users are created
        # with new names
        self.doTestIntegration(
            url=reverse('django_lean.experiments.tests.views.clientsideexperiment_test',
                        args=[self.other_experiment.name]),
            client_factory=lambda i: create_registered_user_client(1000 + i))
    
    def doTestIntegration(self, url, client_factory):
        confirm_human_url = reverse('django_lean.experiments.views.confirm_human')
        found_control = False
        found_test = False
        for i in range(100):
            client = client_factory(i)
            response = client.get(confirm_human_url)
            self.assertEquals(204, response.status_code)
            response = client.get(url)
            self.assertEquals(200, response.status_code)
            in_test = "test" in response.content.lower()
            in_control = "control" in response.content.lower()
            self.assertTrue(in_test != in_control)
            found_control = found_control or in_control
            found_test = found_test or in_test
        
        self.assertTrue(found_control)
        self.assertTrue(found_test)
    
    def testIllegalGroupName(self):
        self.doRenderExperiment("username", "foo group",
                                expect_render_exception=True)
    
    def testBadSyntax(self):
        self.doRenderExperimentToken(
            "username", ("experiment", self.experiment.name, "group", "yeah"),
            expect_parse_exception=True)
    
    def doRenderExperimentToken(self, username, token_tuple,
                           expect_parse_exception=False,
                           expect_render_exception=False):
        internal_render_result = "rendered"
        parser = self.mox.CreateMockAnything()
        child_node_list = self.mox.CreateMockAnything()
        context = {}
        user_factory = self.mox.CreateMockAnything()
        token = self.mox.CreateMockAnything()
        token.split_contents().AndReturn(token_tuple)
        parser.parse(('endexperiment', )).MultipleTimes().AndReturn(
            child_node_list)
        parser.delete_first_token().MultipleTimes()
        user_factory.create_user(context).MultipleTimes().AndReturn(
            TestUser(username=username))
        child_node_list.render(context).MultipleTimes().AndReturn(
            internal_render_result)
        
        self.mox.ReplayAll()
        
        # HACK: there is no way to make a call optional
        child_node_list.render(context)
        parser.parse(('endexperiment', ))
        parser.delete_first_token()
        user_factory.create_user(context)

        
        do_parse = lambda: experiment(parser, token, user_factory=user_factory)
        if expect_parse_exception:
            self.assertRaises(Exception, do_parse)
        else:
            node = do_parse()
        
        if expect_parse_exception:
            self.mox.VerifyAll()
            return None
        
        do_render = lambda: node.render(context)
        if expect_render_exception:
            self.assertRaises(Exception, do_render)
        else:
            actual_render_result = do_render()
        
        if expect_render_exception:
            self.mox.VerifyAll()
            return None
        
        self.mox.VerifyAll()
        
        self.assertTrue(actual_render_result == "" or
                        actual_render_result == internal_render_result)
        in_group = (actual_render_result == internal_render_result)
        return in_group
    
    def doRenderExperiment(self, username, groupname, **kwargs):
        return self.doRenderExperimentToken(
            username, ("experiment", self.experiment.name, groupname),
            **kwargs)
    
    def testExperimentTag(self):
        found_test = False
        found_control = False
        
        for i in range(100):
            username = "******" % i
            in_test = self.doRenderExperiment(username, "test")
            found_test = found_test or in_test
            found_control = found_control or (not in_test)
            
            # make sure we get the same result from a second call
            self.assertEquals(in_test,
                              self.doRenderExperiment(username, "test"))
            
            # make sure that the result of calling control is the opposite of
            # the result of calling test
            in_control = self.doRenderExperiment(username, "control")
            self.assertEquals(not in_test, in_control)
        
        self.assertTrue(found_control and found_test)
    
    def doRenderClientSideExperiment(self, context, username, experiment_name):
        parser = self.mox.CreateMockAnything()
        user_factory = self.mox.CreateMockAnything()
        token = self.mox.CreateMockAnything()
        token.split_contents().AndReturn(("clientsideexperiment",
                                          experiment_name))
        user_factory.create_user(context).MultipleTimes().AndReturn(
            TestUser(username=username))
        
        self.mox.ReplayAll()
        
        # HACK this is the only way to make a call optional
        user_factory.create_user(context)
        
        node = clientsideexperiment(parser, token, user_factory=user_factory)
        node.render(context)
        
        self.mox.VerifyAll()
    
    def testClientSideExperimentTag(self):
        for i in range(100):
            user = User(username="******" % i, email="*****@*****.**" % i)
            user.save()
            
            # create a context for a single "request"
            c = Context({"user": user})
            
            self.doRenderClientSideExperiment(c, "user%i" % i,
                                              self.experiment.name)
            self.assertTrue("client_side_experiments" in c)
            self.doRenderClientSideExperiment(c, "user%i" % i,
                                              self.experiment.name)
            self.doRenderClientSideExperiment(c, "user%i" % i,
                                              self.other_experiment.name)
            self.doRenderClientSideExperiment(c, "user%i" % i,
                                              self.other_experiment.name)
            
            # determine what group the user was assigned to in each experiment
            experiment_partcipant = Participant.objects.get(
                                        user=user,
                                        experiment=self.experiment)
            other_experiment_partcipant = Participant.objects.get(
                                            user=user,
                                            experiment=self.other_experiment)
            
            # assert that exactly two entries were populated in the
            # 'client_side_experiments' dict in the context and that they are
            # equal to the names of our two experiments
            self.assertEquals(2, len(c['client_side_experiments'].keys()))
            self.assertTrue(self.experiment.name in
                               c['client_side_experiments'])
            self.assertTrue(self.other_experiment.name in
                               c['client_side_experiments'])
            
            # get the values that were put in the client_side_experiments dict
            group_name = c['client_side_experiments'][self.experiment.name]
            other_group_name = \
                c['client_side_experiments'][self.other_experiment.name]
            
            # make sure that the values correctly represent the user's
            # enrollments
            group_id = experiment_partcipant.group
            other_group_id = other_experiment_partcipant.group
            self.assertEqual(group_id == Participant.TEST_GROUP,
                             group_name == "test")
            
            self.assertEqual(group_id == Participant.CONTROL_GROUP,
                             group_name == "control")
            
            self.assertEqual(
                        other_group_id == Participant.TEST_GROUP,
                        other_group_name == "test")
            
            self.assertEqual(
                        other_group_id == Participant.CONTROL_GROUP,
                        other_group_name == "control")