def testConversionReportGenerator(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)])
calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEqual(results.count(), 5)
report_days = [ d.date() for d in days[2:7]]
for i in range(5):
self.assertEqual(results[i].date, report_days[4-i])
# Day 2
self.assertEqual(12, results[4].test_group_size)
self.assertEqual(7, results[4].control_group_size)
self.assertEqual(5, results[4].overall_test_conversion)
self.assertEqual(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[0])[0].test_conversion
self.assertEqual(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[2])[0].control_conversion
self.assertEqual(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEqual(5, results[3].test_group_size)
# Day 4
self.assertEqual(0, results[2].test_group_size)
self.assertEqual(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2],
goal_type=goal_types[0])[0].confidence
self.assertEqual(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1],
goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testConversionReportGenerator(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment,
day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(
data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants,
day.date()).InAnyOrder().AndReturn(
data[group]["conversions"][int(
goal_type.name)])
calculator(None, mock_participants,
day.date()).InAnyOrder().AndReturn(
sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(
calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEquals(results.count(), 5)
report_days = [d.date() for d in days[2:7]]
for i in range(5):
self.assertEquals(results[i].date, report_days[4 - i])
# Day 2
self.assertEquals(12, results[4].test_group_size)
self.assertEquals(7, results[4].control_group_size)
self.assertEquals(5, results[4].overall_test_conversion)
self.assertEquals(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[0])[0].test_conversion
self.assertEquals(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[2])[0].control_conversion
self.assertEquals(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEquals(5, results[3].test_group_size)
# Day 4
self.assertEquals(0, results[2].test_group_size)
self.assertEquals(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2], goal_type=goal_types[0])[0].confidence
self.assertEquals(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1], goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029,
day_5_goal_0_confidence,
places=6)
def testExperimentGroupParticipantFinder(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)]
experiment_participant_groups = [
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEqual(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEqual(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEqual(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEqual(2, len(ex2day5testvisitors))
def testExperimentGroupParticipantFinder(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [
AnonymousVisitor.objects.create() for i in range(10)
]
experiment_participant_groups = [
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP,
experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEquals(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(
Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEquals(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(
Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEquals(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(
Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEquals(2, len(ex2day5testvisitors))
