def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
anonymous_visitor = AnonymousVisitor()
anonymous_visitor.save()
anonymous_participant = Participant(
anonymous_visitor=anonymous_visitor,
experiment=self.experiment,
group=Participant.TEST_GROUP)
anonymous_participant.save()
anonymous_participant.enrollment_date = self.experiment.start_date
anonymous_participant.save()
self.other_experiment = Experiment(name="test_experiment2")
self.other_experiment.save()
self.other_experiment.state = Experiment.DISABLED_STATE
self.other_experiment.save()
self.other_experiment.start_date = (date.today() - timedelta(days=7))
self.other_experiment.end_date = (date.today() - timedelta(days=3))
self.other_experiment.save()
def testDisabledExperiment(self):
# disabled test (unenrolled user)
experiment = Experiment(name="disabled")
experiment.save()
user = TestUser(username="******")
self.assertFalse(Experiment.test("disabled", user))
self.assertTrue(Experiment.control("disabled", user))
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
anonymous_visitor = AnonymousVisitor()
anonymous_visitor.save()
anonymous_participant = Participant(anonymous_visitor=anonymous_visitor,
experiment=self.experiment,
group=Participant.TEST_GROUP)
anonymous_participant.save()
anonymous_participant.enrollment_date = self.experiment.start_date
anonymous_participant.save()
self.other_experiment = Experiment(name="test_experiment2")
self.other_experiment.save()
self.other_experiment.state = Experiment.DISABLED_STATE
self.other_experiment.save()
self.other_experiment.start_date = (date.today() -
timedelta(days=7))
self.other_experiment.end_date = (date.today() -
timedelta(days=3))
self.other_experiment.save()
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.other_experiment = Experiment(name="other_test_experiment")
self.other_experiment.save()
self.other_experiment.state = Experiment.ENABLED_STATE
self.other_experiment.save()
self.mox = mox.Mox()
def testUnknownExperiment(self):
# unknown test (unenrolled user)
user = TestUser(username="******")
# unit tests are always run in DEBUG=False by Django
self.assertFalse(Experiment.test("undefined", user))
self.assertTrue(Experiment.control("undefined", user))
try:
settings.DEBUG = True
self.assertRaises(Exception,
lambda: Experiment.test("undefined", user))
self.assertRaises(Exception,
lambda: Experiment.control("undefined", user))
finally:
settings.DEBUG = False
def testPromotedExperiment(self):
# promoted test (unenrolled user)
experiment = Experiment(name="promoted")
experiment.save()
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
user = TestUser(username="******")
self.assertTrue(Experiment.test("promoted", user))
self.assertFalse(Experiment.control("promoted", user))
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
class TestManagement(TestCase):
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
def testManageCommand(self):
with patch(settings, 'LEAN_ENGAGEMENT_CALCULATOR',
'easy_split.experiments.testsettings.SimpleEngagementCalculator'):
#make sure the manage.py command that generates daily stats work
#Running with arguments should raise Exception
self.assertRaises(CommandError,
update_experiment_reports.Command().handle,
"some", "args")
#This is what manage.py will call
self.runner = update_experiment_reports.Command().run_from_argv
#Run the reports
self.runner(['manage.py', 'update_experiment_reports'])
#Make sure they were generated
self.assertEquals(5, DailyEngagementReport.objects.filter(
experiment=self.experiment).count())
self.assertEquals(5, DailyConversionReport.objects.filter(
experiment=self.experiment).count())
def testGroupSanity(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
for i in range(100):
self.client = Client()
response = self.client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertTrue(response.content.strip().lower() in ("test",
"control"))
def testBotExclusion(self):
experiment = Experiment(name="bot_experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
user = TestUser(verified_human=False)
participants_count = Participant.objects.all().count()
in_test = Experiment.test(experiment.name, user)
self.assertEquals(None, user.get_anonymous_id())
self.assertEquals(participants_count, Participant.objects.all().count())
enrollments = user.get_added_enrollments()
self.assertEquals(len(enrollments.keys()), 1)
self.assertTrue(experiment.name in enrollments.keys())
def setUp(self):
staff_user = User(username="******", email="*****@*****.**",
is_staff=True)
staff_user.save()
staff_user.set_password("staff")
staff_user.save()
# self.assertTrue(self.client.login(username='******',
# password='******'))
self.experiment1 = Experiment(name="experiment 1")
self.experiment1.save()
self.experiment1.state = Experiment.ENABLED_STATE
self.experiment1.save()
self.experiment1.start_date -= timedelta(days=5)
self.experiment1.save()
goal_types = [GoalType.objects.create(name='test_goal_%s' % i)
for i in range(3)]
for i in range(1, 6):
DailyEngagementReport.objects.create(date=days_ago(i),
experiment=self.experiment1,
control_score=3.2,
test_score=2.3,
control_group_size=3,
test_group_size=5,
confidence=93.2)
conversion_report = DailyConversionReport.objects.create(
date=days_ago(i),
experiment=self.experiment1,
overall_test_conversion=12,
overall_control_conversion=9,
test_group_size=39,
control_group_size=27,
confidence=87.4)
for goal_type in goal_types:
DailyConversionReportGoalData.objects.create(
report=conversion_report,
goal_type=goal_type,
test_conversion=11,
control_conversion=7,
confidence=45.3)
self.experiment2 = Experiment(name="experiment 2")
self.experiment2.save()
self.experiment3 = Experiment(name="experiment 3")
self.experiment3.save()
self.experiment3.state = Experiment.ENABLED_STATE
self.experiment3.save()
self.experiment3.state = Experiment.PROMOTED_STATE
self.experiment3.save()
class TestManagement(TestCase):
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
def testManageCommand(self):
with patch(
settings, 'LEAN_ENGAGEMENT_CALCULATOR',
'easy_split.experiments.testsettings.SimpleEngagementCalculator'
):
#make sure the manage.py command that generates daily stats work
#Running with arguments should raise Exception
self.assertRaises(CommandError,
update_experiment_reports.Command().handle,
"some", "args")
#This is what manage.py will call
self.runner = update_experiment_reports.Command().run_from_argv
#Run the reports
self.runner(['manage.py', 'update_experiment_reports'])
#Make sure they were generated
self.assertEquals(
5,
DailyEngagementReport.objects.filter(
experiment=self.experiment).count())
self.assertEquals(
5,
DailyConversionReport.objects.filter(
experiment=self.experiment).count())
def testExperimentGroupParticipantFinder(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)]
experiment_participant_groups = [
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEquals(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEquals(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEquals(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEquals(2, len(ex2day5testvisitors))
def testAnonymousUser(self):
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
for i in range(100):
user = TestUser()
in_test = Experiment.test("enabled", user)
anonymous_id = user.get_anonymous_id()
self.assertNotEquals(None, anonymous_id)
in_control = Experiment.control("enabled", user)
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertEquals(in_control, Experiment.control("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
if in_test:
test_user = user
if in_control:
control_user = user
self.assertTrue(test_user and control_user)
user = TestUser()
experiment = Experiment(name="disabled")
experiment.save()
self.assertFalse(Experiment.test("disabled", user))
self.assertEquals(None, user.get_anonymous_id())
def testEnabledPromotedAndDisabledExperiment(self):
# enabled test, new user (prove we get both results)
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
test_user = None
control_user = None
for i in range(100):
user = TestUser(username="******" % i)
in_test = Experiment.test("enabled", user)
in_control = Experiment.control("enabled", user)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(in_control, Experiment.control("enabled", user))
if in_test:
test_user = user
if in_control:
control_user = user
self.assertNotEquals(None, test_user)
self.assertNotEquals(None, control_user)
self.assertNotEquals(test_user, control_user)
# promoted test with an enrolled user
# (prove we get both results, then that all users are in test)
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
self.assertFalse(Experiment.control("enabled", control_user))
self.assertFalse(Experiment.control("enabled", test_user))
self.assertTrue(Experiment.test("enabled", control_user))
self.assertTrue(Experiment.test("enabled", test_user))
# disabled test with an enrolled user
# (prove we get both results, then that all users are in control)
experiment.state = Experiment.DISABLED_STATE
experiment.save()
self.assertTrue(Experiment.control("enabled", control_user))
self.assertTrue(Experiment.control("enabled", test_user))
self.assertFalse(Experiment.test("enabled", control_user))
self.assertFalse(Experiment.test("enabled", test_user))
def testConversionReportGenerator(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)])
calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEquals(results.count(), 5)
report_days = [ d.date() for d in days[2:7]]
for i in range(5):
self.assertEquals(results[i].date, report_days[4-i])
# Day 2
self.assertEquals(12, results[4].test_group_size)
self.assertEquals(7, results[4].control_group_size)
self.assertEquals(5, results[4].overall_test_conversion)
self.assertEquals(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[0])[0].test_conversion
self.assertEquals(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[2])[0].control_conversion
self.assertEquals(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEquals(5, results[3].test_group_size)
# Day 4
self.assertEquals(0, results[2].test_group_size)
self.assertEquals(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2],
goal_type=goal_types[0])[0].confidence
self.assertEquals(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1],
goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testGetConversionData(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
yesterday = date.today() - timedelta(days=1)
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = yesterday
experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
report = DailyConversionReport.objects.create(experiment=experiment,
date=yesterday,
overall_test_conversion=17,
overall_control_conversion=12,
test_group_size=139,
control_group_size=142,
confidence=87.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[0],
test_conversion=11,
control_conversion=0,
confidence=65.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[1],
test_conversion=0,
control_conversion=21,
confidence=None)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[2],
test_conversion=23,
control_conversion=21,
confidence=100)
data = get_conversion_data(experiment, yesterday)
self.assertEquals(data['date'], yesterday)
self.assertTrue("totals" in data)
self.assertTrue("goal_types" in data)
self.assertEquals(4, len(data["goal_types"].keys()))
for goal_type in goal_types[0:3]:
self.assertTrue(goal_type.name in data["goal_types"])
goal_type_data = data["goal_types"][goal_type.name]
self.assertTrue("test_count" in goal_type_data)
self.assertTrue("control_count" in goal_type_data)
self.assertTrue("test_rate" in goal_type_data)
self.assertTrue("control_rate" in goal_type_data)
self.assertTrue("improvement" in goal_type_data)
self.assertTrue("confidence" in goal_type_data)
self.assertEquals(None, data["goal_types"][goal_types[3].name])
self.assertEquals(139, data["test_group_size"])
self.assertEquals(142, data["control_group_size"])
totals = data["totals"]
expected_test_rate = 17. / 139. * 100.
expected_control_rate = 12. / 142. * 100.
expected_improvement = (expected_test_rate - expected_control_rate) / expected_control_rate * 100.
self.assertAlmostEquals(expected_test_rate, totals["test_rate"])
self.assertAlmostEquals(expected_control_rate, totals["control_rate"])
self.assertAlmostEquals(expected_improvement, totals["improvement"])
self.assertAlmostEquals(87.3, totals["confidence"])
self.assertEquals(17, totals["test_count"])
self.assertEquals(12, totals["control_count"])
self.assertEquals(0, data["goal_types"][goal_types[0].name]["control_rate"])
self.assertAlmostEquals(11./139*100., data["goal_types"][goal_types[0].name]["test_rate"])
self.assertEquals(None, data["goal_types"][goal_types[0].name]["improvement"])
self.assertAlmostEquals(65.3, data["goal_types"][goal_types[0].name]["confidence"])
self.assertEquals(11, data["goal_types"][goal_types[0].name]["test_count"])
self.assertEquals(0, data["goal_types"][goal_types[0].name]["control_count"])
self.assertAlmostEquals(21./142*100., data["goal_types"][goal_types[1].name]["control_rate"])
self.assertEquals(None, data["goal_types"][goal_types[1].name]["confidence"])
self.assertEquals(None, data["goal_types"][goal_types[1].name]["improvement"])
self.assertAlmostEquals((23./139-21./142)/(21./142)*100.,
data["goal_types"][goal_types[2].name]["improvement"])
def testConversionReportGenerator(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment,
day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(
data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants,
day.date()).InAnyOrder().AndReturn(
data[group]["conversions"][int(
goal_type.name)])
calculator(None, mock_participants,
day.date()).InAnyOrder().AndReturn(
sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(
calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEquals(results.count(), 5)
report_days = [d.date() for d in days[2:7]]
for i in range(5):
self.assertEquals(results[i].date, report_days[4 - i])
# Day 2
self.assertEquals(12, results[4].test_group_size)
self.assertEquals(7, results[4].control_group_size)
self.assertEquals(5, results[4].overall_test_conversion)
self.assertEquals(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[0])[0].test_conversion
self.assertEquals(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[2])[0].control_conversion
self.assertEquals(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEquals(5, results[3].test_group_size)
# Day 4
self.assertEquals(0, results[2].test_group_size)
self.assertEquals(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2], goal_type=goal_types[0])[0].confidence
self.assertEquals(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1], goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029,
day_5_goal_0_confidence,
places=6)
def testParticipants(self):
user1 = User(username="******", email="*****@*****.**")
user1.save()
user2 = User(username="******", email="*****@*****.**")
user2.save()
user3 = User(username="******", email="*****@*****.**")
user3.save()
anonymous_visitor1 = AnonymousVisitor()
anonymous_visitor1.save()
anonymous_visitor2 = AnonymousVisitor()
anonymous_visitor2.save()
anonymous_visitor3 = AnonymousVisitor()
anonymous_visitor3.save()
experiment1 = Experiment(name="Experiment 1")
experiment1.save()
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
participant11 = Participant(user=user1,
experiment= experiment1,
group=Participant.CONTROL_GROUP)
participant11.save()
participant12 = Participant(user=user1,
experiment= experiment2,
group=Participant.TEST_GROUP)
participant12.save()
participant21 = Participant(user=user2,
experiment= experiment1,
group=Participant.CONTROL_GROUP)
participant21.save()
participant22 = Participant(user=user2,
experiment= experiment2,
group=Participant.CONTROL_GROUP)
participant22.save()
self.assertRaises(Exception,
lambda: Participant(user=user2,
experiment= experiment2,
group=Participant.TEST_GROUP).save())
# create anonymous participants
participant31 = Participant(anonymous_visitor=anonymous_visitor1,
experiment=experiment1,
group=Participant.CONTROL_GROUP)
participant31.save()
participant32 = Participant(anonymous_visitor=anonymous_visitor1,
experiment=experiment2,
group=Participant.TEST_GROUP)
participant32.save()
participant42 = Participant(anonymous_visitor=anonymous_visitor2,
experiment=experiment2,
group=Participant.CONTROL_GROUP)
participant42.save()
self.assertRaises(Exception,
lambda: Participant(anonymous_visitor=anonymous_visitor1,
experiment=experiment1,
group=Participant.TEST_GROUP).save())
self.assertRaises(Exception,
lambda: Participant(experiment=experiment1,
group=Participant.TEST_GROUP).save())
self.assertRaises(Exception,
lambda: Participant(user=user3,
anonymous_visitor=anonymous_visitor3,
experiment=experiment1,
group=Participant.TEST_GROUP))
self.assertRaises(Exception,
lambda: Participant(user=user3,
experiment=experiment1).save())
self.assertNotEquals(participant11.enrollment_date, None)
participant11.enrollment_date = None
self.assertRaises(Exception,
lambda: participant11.save())
participant12.group = None
self.assertRaises(Exception,
lambda: participant12.save())
def testExperimentStates(self):
experiment1 = Experiment(name="test_experiment_1")
experiment1.save()
experiment1 = Experiment.objects.get(name="test_experiment_1")
self.assertRaises(Exception,
lambda: Experiment(name="test_experiment_1").save())
#test default values
self.assertEquals(experiment1.state, Experiment.DISABLED_STATE)
self.assertEquals(experiment1.start_date, None)
self.assertEquals(experiment1.end_date, None)
#enable the experiment
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
self.assertNotEquals(experiment1.start_date, None)
self.assertEquals(experiment1.end_date, None)
#disable the experiement
old_start_date = experiment1.start_date
experiment1.state = Experiment.DISABLED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertNotEquals(experiment1.end_date, None)
#enable the experiment
old_end_date = experiment1.end_date
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertEquals(experiment1.end_date, old_end_date)
#promote the experiment
experiment1.state = Experiment.PROMOTED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertEquals(experiment1.end_date, old_end_date)
experiment2 = Experiment(name= "Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
self.assertEquals(experiment2.start_date, experiment1.start_date)
experiment2.state = Experiment.PROMOTED_STATE
experiment2.save()
self.assertNotEquals(experiment2.start_date, None)
self.assertNotEquals(experiment2.end_date, None)
def testParticipantEnrollment(self):
experiment1 = Experiment(name="Experiment 1")
experiment1.save()
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
for i in range(1000):
username="******" % i
in_test1 = Experiment.test(experiment1.name, TestUser(username=username))
self.assertEquals(in_test1, not Experiment.control(experiment1.name,
TestUser(username=username)))
if in_test1:
num_test1 += 1
else:
num_control1 += 1
in_test2 = not Experiment.control(experiment2.name, TestUser(username=username))
self.assertEquals(in_test2, Experiment.test(experiment2.name,
TestUser(username=username)))
if in_test2:
num_test2 += 1
else:
num_control2 += 1
self.assert_(num_control1 > 400)
self.assert_(num_control2 > 400)
self.assert_(num_test1 > 400)
self.assert_(num_test2 > 400)
class TestExperimentViews(object):
urls = 'experiments.tests.urls'
def setUp(self):
staff_user = User(username="******", email="*****@*****.**",
is_staff=True)
staff_user.save()
staff_user.set_password("staff")
staff_user.save()
# self.assertTrue(self.client.login(username='******',
# password='******'))
self.experiment1 = Experiment(name="experiment 1")
self.experiment1.save()
self.experiment1.state = Experiment.ENABLED_STATE
self.experiment1.save()
self.experiment1.start_date -= timedelta(days=5)
self.experiment1.save()
goal_types = [GoalType.objects.create(name='test_goal_%s' % i)
for i in range(3)]
for i in range(1, 6):
DailyEngagementReport.objects.create(date=days_ago(i),
experiment=self.experiment1,
control_score=3.2,
test_score=2.3,
control_group_size=3,
test_group_size=5,
confidence=93.2)
conversion_report = DailyConversionReport.objects.create(
date=days_ago(i),
experiment=self.experiment1,
overall_test_conversion=12,
overall_control_conversion=9,
test_group_size=39,
control_group_size=27,
confidence=87.4)
for goal_type in goal_types:
DailyConversionReportGoalData.objects.create(
report=conversion_report,
goal_type=goal_type,
test_conversion=11,
control_conversion=7,
confidence=45.3)
self.experiment2 = Experiment(name="experiment 2")
self.experiment2.save()
self.experiment3 = Experiment(name="experiment 3")
self.experiment3.save()
self.experiment3.state = Experiment.ENABLED_STATE
self.experiment3.save()
self.experiment3.state = Experiment.PROMOTED_STATE
self.experiment3.save()
def testListExperimentsView(self):
url = reverse('experiments.views.list_experiments')
response = self.client.get(url)
self.assertEquals(response.status_code, 200)
header, data = get_tables(response.content)[0]
data.sort()
self.assertEquals([u'Name', u'Status', u'Start Date', u'End Date'],
header)
self.assertEquals([[u'experiment 1', u'Enabled',
unicode(self.experiment1.start_date), u'None'],
[u'experiment 2', u'Disabled', u'None', u'None'],
[u'experiment 3', u'Promoted',
unicode(self.experiment3.start_date),
unicode(self.experiment3.end_date)]],
data)
def testAnonymousUserCannotAccessReports(self):
self.client.logout()
url = reverse('experiments.views.experiment_details',
args=['experiment 1'])
response = self.client.get(url)
self.assertEquals(response.status_code, 200)
self.assertTrue('<title>Log in' in response.content)
url = reverse('experiments.views.list_experiments')
response = self.client.get(url)
self.assertEquals(response.status_code, 200)
self.assertTrue('<title>Log in' in response.content)
def testRegularUserCannotAccessReports(self):
self.client.logout()
regular_user = User(username="******", email="*****@*****.**")
regular_user.save()
regular_user.set_password("joe")
regular_user.save()
self.assertTrue(self.client.login(username='******',
password='******'))
url = reverse('experiments.views.experiment_details',
args=['experiment 1'])
response = self.client.get(url)
self.assertEquals(response.status_code, 200)
self.assertTrue('<title>Log in' in response.content)
url = reverse('experiments.views.list_experiments')
response = self.client.get(url)
self.assertEquals(response.status_code, 200)
self.assertTrue('<title>Log in' in response.content)
def test404IfExperimentDoesntExist(self):
url = reverse('experiments.views.experiment_details',
args=['inexistant experiment'])
response = self.client.get(url)
self.assertEquals(response.status_code, 404)
def testExperimentDetailsView(self):
url = reverse('experiments.views.experiment_details',
args=['experiment 1'])
response = self.client.get(url)
self.assertEquals(response.status_code, 200)
tables = get_tables(response.content)
experiment_properties = tables.pop(0)
self.assertEquals([u'Property', u'Value'],
experiment_properties[0])
self.assertEquals([[u'Experiment Name', u'experiment 1'],
[u'Status', u'Enabled'],
[u'Start Date',
unicode(self.experiment1.start_date)],
[u'End Date', u'None']],
experiment_properties[1])
conversion_summary = tables.pop(0)
self.assertEquals([u' ', u'Control Group', u'Test Group',
u'Improvement', u'Confidence'],
conversion_summary[0])
self.assertEquals([[u'Participants', u'27', u'39', u' ',
u' '],
[u'test_goal_0', u'7 (25.9 %)', u'11 (28.2 %)',
u'8 %', u'45 %'],
[u'test_goal_1', u'7 (25.9 %)', u'11 (28.2 %)',
u'8 %', u'45 %'],
[u'test_goal_2', u'7 (25.9 %)', u'11 (28.2 %)',
u'8 %', u'45 %'],
[u'Any', u'9 (33.3 %)', u'12 (30.8 %)', u'-7 %',
u'87 %']],
conversion_summary[1])
engagement_summary = tables.pop(0)
self.assertEquals([u' ', u'Control Group', u'Test Group',
u'Improvement', u'Confidence'],
engagement_summary[0])
self.assertEquals([[u'Participants', u'3', u'5', u' ', u' '],
[u'Engagement Score', u'3.2', u'2.3', u'-28 %',
u'93 %']],
engagement_summary[1])
conversion_details = tables.pop(0)
self.assertTrue(conversion_details[0])
self.assertTrue(conversion_details[1])
engagement_details = tables.pop(0)
self.assertEquals([u'Report Date', u'Control Group Size',
u'Control Group Score', u'Test Group Size',
u'Test Group Score', u'Test Group Improvement',
u'Confidence'],
engagement_details[0])
self.assertEquals([unicode(days_ago(1)), u'3', u'3.2', u'5', u'2.3',
u'-28 %', u'93 %'],
engagement_details[1][0])
self.assertEquals([], tables)
def testVerifyHuman(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
other_experiment = Experiment(name="other_experiment")
other_experiment.save()
other_experiment.state = Experiment.ENABLED_STATE
other_experiment.save()
self.client = Client()
original_participants_count = Participant.objects.all().count()
original_anonymous_visitors_count = AnonymousVisitor.objects.all().count()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
response = self.client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(original_participants_count,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count,
AnonymousVisitor.objects.all().count())
confirm_human_url = reverse('experiments.views.confirm_human')
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count+1,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
# calling it again does nothing
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count+1,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
other_experiment_url = reverse(
'experiments.tests.views.experiment_test',
args=[other_experiment.name])
response = self.client.get(other_experiment_url)
self.assertEquals(response.status_code, 200)
# a new participant is immediately created for the new experiment
self.assertEquals(original_participants_count + 2,
Participant.objects.all().count())
# the existing anonymous visitor is reused
self.assertEquals(original_anonymous_visitors_count + 1,
AnonymousVisitor.objects.all().count())
# the new experiment does not cause storage of a temporary enrollment
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count+2,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
def testGroupSanity(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
for i in range(100):
self.client = Client()
response = self.client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertTrue(response.content.strip().lower() in ("test",
"control"))
def testPngResponse(self):
experiment = Experiment(name="test-experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
goal_type = GoalType(name='test-goal')
goal_type.save()
experiment_url = reverse("easy_split.experiments.tests.views.experiment_test",
args=[experiment.name])
confirm_human_url = reverse("easy_split.experiments.views.confirm_human")
client = Client()
# we can call with invalid or inexisting names, the response is the same
self.call_goal(client, '')
self.call_goal(client, 'unknown-goal')
# this is an anonymous visitor not enrolled in an experiment,
# so no records should be created
self.call_goal(client, goal_type.name)
self.assertEquals(0, GoalRecord.objects.filter(goal_type=goal_type).count())
nb_anonymous_visitors = AnonymousVisitor.objects.count()
# force the user to be a verified human
response = client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
# force the anonymous visitor to be enrolled in an experiment
response = client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
client.get('/test-experiment/%s' % experiment.name)
self.assertEquals(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
# now call an existing goal again - it should be recorded
self.call_goal(client, goal_type.name)
self.assertEquals(1, GoalRecord.objects.filter(goal_type=goal_type).count())
# should be recorded again
self.call_goal(client, goal_type.name)
self.assertEquals(2, GoalRecord.objects.filter(goal_type=goal_type).count())
# validate that both of the records have the same anonymous_visitor
two_goal_records = GoalRecord.objects.filter(goal_type=goal_type)
self.assertEquals(two_goal_records[0].anonymous_visitor,
two_goal_records[1].anonymous_visitor)
# try it with a registered user
client = Client()
user = User(username="******", email="*****@*****.**")
user.set_password("password")
user.save()
response = client.login(username=user.username, password='******')
self.assertTrue(response)
# force the registered user to be enrolled in an experiment
client.get('/test-experiment/%s' % experiment.name)
self.call_goal(client, goal_type.name)
# since the user was registered, no new records should be created
self.assertEquals(2, GoalRecord.objects.filter(goal_type=goal_type).count())
def testPngResponse(self):
experiment = Experiment(name="test-experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
goal_type = GoalType(name='test-goal')
goal_type.save()
experiment_url = reverse(
"easy_split.experiments.tests.views.experiment_test",
args=[experiment.name])
confirm_human_url = reverse(
"easy_split.experiments.views.confirm_human")
client = Client()
# we can call with invalid or inexisting names, the response is the same
self.call_goal(client, '')
self.call_goal(client, 'unknown-goal')
# this is an anonymous visitor not enrolled in an experiment,
# so no records should be created
self.call_goal(client, goal_type.name)
self.assertEquals(
0,
GoalRecord.objects.filter(goal_type=goal_type).count())
nb_anonymous_visitors = AnonymousVisitor.objects.count()
# force the user to be a verified human
response = client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
# force the anonymous visitor to be enrolled in an experiment
response = client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
client.get('/test-experiment/%s' % experiment.name)
self.assertEquals(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
# now call an existing goal again - it should be recorded
self.call_goal(client, goal_type.name)
self.assertEquals(
1,
GoalRecord.objects.filter(goal_type=goal_type).count())
# should be recorded again
self.call_goal(client, goal_type.name)
self.assertEquals(
2,
GoalRecord.objects.filter(goal_type=goal_type).count())
# validate that both of the records have the same anonymous_visitor
two_goal_records = GoalRecord.objects.filter(goal_type=goal_type)
self.assertEquals(two_goal_records[0].anonymous_visitor,
two_goal_records[1].anonymous_visitor)
# try it with a registered user
client = Client()
user = User(username="******", email="*****@*****.**")
user.set_password("password")
user.save()
response = client.login(username=user.username, password='******')
self.assertTrue(response)
# force the registered user to be enrolled in an experiment
client.get('/test-experiment/%s' % experiment.name)
self.call_goal(client, goal_type.name)
# since the user was registered, no new records should be created
self.assertEquals(
2,
GoalRecord.objects.filter(goal_type=goal_type).count())
class TestDailyReports(TestCase):
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
anonymous_visitor = AnonymousVisitor()
anonymous_visitor.save()
anonymous_participant = Participant(
anonymous_visitor=anonymous_visitor,
experiment=self.experiment,
group=Participant.TEST_GROUP)
anonymous_participant.save()
anonymous_participant.enrollment_date = self.experiment.start_date
anonymous_participant.save()
self.other_experiment = Experiment(name="test_experiment2")
self.other_experiment.save()
self.other_experiment.state = Experiment.DISABLED_STATE
self.other_experiment.save()
self.other_experiment.start_date = (date.today() - timedelta(days=7))
self.other_experiment.end_date = (date.today() - timedelta(days=3))
self.other_experiment.save()
def testDailyEngagementReport(self):
users_test = []
users_control = []
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
#create users
for i in range(5):
users_control.append(
create_user_in_group(self.experiment, i,
Participant.CONTROL_GROUP,
date.today() - timedelta(days=i)))
users_test.append(
create_user_in_group(self.experiment, i,
Participant.TEST_GROUP,
date.today() - timedelta(days=i)))
# users_<test|control>[0] were enrolled today, [1] 1 day ago, etc.
report_date = date.today() - timedelta(days=1)
expected_engagement_score_calls = {
(users_test[1], date.today() - timedelta(days=1), report_date):
3.2,
(users_test[2], date.today() - timedelta(days=2), report_date):
2.5,
(users_test[3], date.today() - timedelta(days=3), report_date):
4.1,
(users_test[4], date.today() - timedelta(days=4), report_date): 0,
(users_control[1], date.today() - timedelta(days=1), report_date):
0,
(users_control[2], date.today() - timedelta(days=2), report_date):
0,
(users_control[3], date.today() - timedelta(days=3), report_date):
0,
(users_control[4], date.today() - timedelta(days=4), report_date):
0
}
test_case = self
class EngagementScoreCalculatorStub(object):
def calculate_user_engagement_score(self, user, start_date,
end_date):
test_case.assertNotEquals(user, None)
test_case.assertTrue(
expected_engagement_score_calls.has_key(
(user, start_date, end_date)))
return expected_engagement_score_calls[(user, start_date,
end_date)]
(EngagementReportGenerator(EngagementScoreCalculatorStub()).
generate_daily_report_for_experiment(self.experiment, report_date))
experiment_report = DailyEngagementReport.objects.get(
experiment=self.experiment, date=report_date)
self.assertAlmostEqual((3.2 + 2.5 + 4.1 + 0) / 4.0,
experiment_report.test_score)
self.assertAlmostEqual(0.0, experiment_report.control_score)
self.assertEquals(4, experiment_report.test_group_size)
self.assertEquals(4, experiment_report.control_group_size)
self.assertAlmostEqual(96.819293337188498,
experiment_report.confidence)
def testZeroParticipantExperiment(self):
mocker = mox.Mox()
engagement_calculator = mocker.CreateMockAnything()
mocker.ReplayAll()
report_date = date.today()
EngagementReportGenerator(
engagement_score_calculator=engagement_calculator
).generate_daily_report_for_experiment(self.other_experiment,
report_date)
experiment_report = DailyEngagementReport.objects.get(
experiment=self.other_experiment, date=report_date)
mocker.VerifyAll()
self.assertEquals(None, experiment_report.test_score)
self.assertEquals(None, experiment_report.control_score)
self.assertEquals(0, experiment_report.test_group_size)
self.assertEquals(0, experiment_report.control_group_size)
def testGenerateAllDailyEngagementReports(self):
class DummyEngagementCalculator(object):
def calculate_user_engagement_score(self, user, start_date,
end_date):
return 7
engagement_report_generator = EngagementReportGenerator(
engagement_score_calculator=DummyEngagementCalculator())
engagement_report_generator.generate_daily_report_for_experiment(
self.experiment,
date.today() - timedelta(days=2))
engagement_report_generator.generate_daily_report_for_experiment(
self.experiment,
date.today() - timedelta(days=3))
engagement_report_generator.generate_all_daily_reports()
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() -
timedelta(days=1))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() -
timedelta(days=2))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() -
timedelta(days=3))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() -
timedelta(days=4))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() -
timedelta(days=5))
self.assertEquals(
5,
DailyEngagementReport.objects.filter(
experiment=self.experiment).count())
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() -
timedelta(days=3))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() -
timedelta(days=4))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() -
timedelta(days=5))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() -
timedelta(days=6))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() -
timedelta(days=7))
self.assertEquals(
5,
DailyEngagementReport.objects.filter(
experiment=self.other_experiment).count())
def create_goal_record(self, record_datetime, anonymous_visitor,
goal_type):
record = GoalRecord.objects.create(anonymous_visitor=anonymous_visitor,
goal_type=goal_type)
record.created = record_datetime
record.save()
def create_participant(self, anonymous_visitor, experiment,
enrollment_date, group):
participant = Participant.objects.create(
anonymous_visitor=anonymous_visitor,
experiment=experiment,
group=group)
participant.enrollment_date = enrollment_date
participant.save()
return participant
def testParticipantConversionCalculator(self):
goal_types = [GoalType.objects.create(name=str(i)) for i in range(3)]
anonymous_visitor = AnonymousVisitor.objects.create()
participant = self.create_participant(
anonymous_visitor=anonymous_visitor,
experiment=self.experiment,
enrollment_date=self.experiment.start_date + timedelta(days=2),
group=Participant.TEST_GROUP)
days = [
datetime.combine(self.experiment.start_date + timedelta(days=i),
time(hour=12)) for i in range(5)
]
nb_goal_records = GoalRecord.objects.all().count()
self.create_goal_record(days[0], anonymous_visitor, goal_types[0])
self.create_goal_record(days[0], anonymous_visitor, goal_types[1])
self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
self.create_goal_record(days[2], anonymous_visitor, goal_types[1])
self.create_goal_record(days[3], anonymous_visitor, goal_types[0])
self.create_goal_record(days[4], anonymous_visitor, goal_types[0])
self.create_goal_record(days[4], anonymous_visitor, goal_types[0])
self.assertEquals(nb_goal_records + 8,
GoalRecord.objects.all().count())
# wasn't enrolled yet!
self.assertEquals(
calculate_participant_conversion(participant, goal_types[0],
days[0]), 0)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[1],
days[0]), 0)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[2],
days[0]), 0)
self.assertEquals(
calculate_participant_conversion(participant, None, days[0]), 0)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[0],
days[1]), 0)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[1],
days[1]), 0)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[2],
days[1]), 0)
self.assertEquals(
calculate_participant_conversion(participant, None, days[1]), 0)
# now enrolled
self.assertEquals(
calculate_participant_conversion(participant, goal_types[0],
days[2]), 0)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[1],
days[2]), 1)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[2],
days[2]), 0)
# "any" is one
self.assertEquals(
calculate_participant_conversion(participant, None, days[2]), 1)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[0],
days[3]), 1)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[1],
days[3]), 1)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[2],
days[3]), 0)
# "any" is one, even though two different goals were achieved
self.assertEquals(
calculate_participant_conversion(participant, None, days[3]), 1)
# there were three conversions on this day for goal 0, but we only count the first!
self.assertEquals(
calculate_participant_conversion(participant, goal_types[0],
days[4]), 1)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[1],
days[4]), 1)
self.assertEquals(
calculate_participant_conversion(participant, goal_types[2],
days[4]), 0)
self.assertEquals(
calculate_participant_conversion(participant, None, days[4]), 1)
def testGoalTypeConversionCalculator(self):
mocker = mox.Mox()
participants = [
mocker.CreateMockAnything(),
mocker.CreateMockAnything(),
mocker.CreateMockAnything()
]
goal_type = mocker.CreateMockAnything()
report_date = mocker.CreateMockAnything()
participant_conversion_calculator = mocker.CreateMockAnything()
participant_conversion_calculator(
participants[0], goal_type, report_date).InAnyOrder().AndReturn(1)
participant_conversion_calculator(
participants[1], goal_type, report_date).InAnyOrder().AndReturn(0)
participant_conversion_calculator(
participants[2], goal_type, report_date).InAnyOrder().AndReturn(1)
mocker.ReplayAll()
self.assertEquals(
2,
calculate_goal_type_conversion(goal_type, participants,
report_date,
participant_conversion_calculator))
mocker.VerifyAll()
def testExperimentGroupParticipantFinder(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [
AnonymousVisitor.objects.create() for i in range(10)
]
experiment_participant_groups = [
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP,
experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEquals(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(
Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEquals(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(
Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEquals(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(
Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEquals(2, len(ex2day5testvisitors))
def testGetConversionData(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
yesterday = date.today() - timedelta(days=1)
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = yesterday
experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
report = DailyConversionReport.objects.create(
experiment=experiment,
date=yesterday,
overall_test_conversion=17,
overall_control_conversion=12,
test_group_size=139,
control_group_size=142,
confidence=87.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[0],
test_conversion=11,
control_conversion=0,
confidence=65.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[1],
test_conversion=0,
control_conversion=21,
confidence=None)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[2],
test_conversion=23,
control_conversion=21,
confidence=100)
data = get_conversion_data(experiment, yesterday)
self.assertEquals(data['date'], yesterday)
self.assertTrue("totals" in data)
self.assertTrue("goal_types" in data)
self.assertEquals(4, len(data["goal_types"].keys()))
for goal_type in goal_types[0:3]:
self.assertTrue(goal_type.name in data["goal_types"])
goal_type_data = data["goal_types"][goal_type.name]
self.assertTrue("test_count" in goal_type_data)
self.assertTrue("control_count" in goal_type_data)
self.assertTrue("test_rate" in goal_type_data)
self.assertTrue("control_rate" in goal_type_data)
self.assertTrue("improvement" in goal_type_data)
self.assertTrue("confidence" in goal_type_data)
self.assertEquals(None, data["goal_types"][goal_types[3].name])
self.assertEquals(139, data["test_group_size"])
self.assertEquals(142, data["control_group_size"])
totals = data["totals"]
expected_test_rate = 17. / 139. * 100.
expected_control_rate = 12. / 142. * 100.
expected_improvement = (expected_test_rate - expected_control_rate
) / expected_control_rate * 100.
self.assertAlmostEquals(expected_test_rate, totals["test_rate"])
self.assertAlmostEquals(expected_control_rate, totals["control_rate"])
self.assertAlmostEquals(expected_improvement, totals["improvement"])
self.assertAlmostEquals(87.3, totals["confidence"])
self.assertEquals(17, totals["test_count"])
self.assertEquals(12, totals["control_count"])
self.assertEquals(
0, data["goal_types"][goal_types[0].name]["control_rate"])
self.assertAlmostEquals(
11. / 139 * 100.,
data["goal_types"][goal_types[0].name]["test_rate"])
self.assertEquals(
None, data["goal_types"][goal_types[0].name]["improvement"])
self.assertAlmostEquals(
65.3, data["goal_types"][goal_types[0].name]["confidence"])
self.assertEquals(11,
data["goal_types"][goal_types[0].name]["test_count"])
self.assertEquals(
0, data["goal_types"][goal_types[0].name]["control_count"])
self.assertAlmostEquals(
21. / 142 * 100.,
data["goal_types"][goal_types[1].name]["control_rate"])
self.assertEquals(None,
data["goal_types"][goal_types[1].name]["confidence"])
self.assertEquals(
None, data["goal_types"][goal_types[1].name]["improvement"])
self.assertAlmostEquals(
(23. / 139 - 21. / 142) / (21. / 142) * 100.,
data["goal_types"][goal_types[2].name]["improvement"])
#TODO test with zero participants and check rate == None
#TODO sometimes confidence cannot be calculated and must return None. Add a test to verify this.
def testConversionReportGenerator(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment,
day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(
data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants,
day.date()).InAnyOrder().AndReturn(
data[group]["conversions"][int(
goal_type.name)])
calculator(None, mock_participants,
day.date()).InAnyOrder().AndReturn(
sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(
calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEquals(results.count(), 5)
report_days = [d.date() for d in days[2:7]]
for i in range(5):
self.assertEquals(results[i].date, report_days[4 - i])
# Day 2
self.assertEquals(12, results[4].test_group_size)
self.assertEquals(7, results[4].control_group_size)
self.assertEquals(5, results[4].overall_test_conversion)
self.assertEquals(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[0])[0].test_conversion
self.assertEquals(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[2])[0].control_conversion
self.assertEquals(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEquals(5, results[3].test_group_size)
# Day 4
self.assertEquals(0, results[2].test_group_size)
self.assertEquals(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2], goal_type=goal_types[0])[0].confidence
self.assertEquals(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1], goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029,
day_5_goal_0_confidence,
places=6)
def testExperimentGroupParticipantFinder(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [
AnonymousVisitor.objects.create() for i in range(10)
]
experiment_participant_groups = [
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP,
experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEquals(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(
Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEquals(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(
Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEquals(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(
Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEquals(2, len(ex2day5testvisitors))
class ExperimentTagsTest(TestCase):
urls = 'easy_split.experiments.tests.urls'
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.other_experiment = Experiment(name="other_test_experiment")
self.other_experiment.save()
self.other_experiment.state = Experiment.ENABLED_STATE
self.other_experiment.save()
self.mox = mox.Mox()
def testIntegrationWithAnonymousVisitor(self):
self.doTestIntegration(
url=reverse('easy_split.experiments.tests.views.experiment_test',
args=[self.experiment.name]),
client_factory=lambda i: Client())
self.doTestIntegration(
url=reverse('easy_split.experiments.tests.views.clientsideexperiment_test',
args=[self.other_experiment.name]),
client_factory=lambda i: Client())
def testIntegrationWithRegisteredUser(self):
def create_registered_user_client(i):
user = User(username="******" % i, email="*****@*****.**" % i)
user.set_password("password")
user.save()
client = Client()
if not client.login(username=user.username, password="******"):
raise Exception("login failure")
return client
self.doTestIntegration(
url=reverse('easy_split.experiments.tests.views.experiment_test',
args=[self.experiment.name]),
client_factory=create_registered_user_client)
# we wrap our factory to ensure that the users are created
# with new names
self.doTestIntegration(
url=reverse('easy_split.experiments.tests.views.clientsideexperiment_test',
args=[self.other_experiment.name]),
client_factory=lambda i: create_registered_user_client(1000 + i))
def doTestIntegration(self, url, client_factory):
confirm_human_url = reverse('easy_split.experiments.views.confirm_human')
found_control = False
found_test = False
for i in range(100):
client = client_factory(i)
response = client.get(confirm_human_url)
self.assertEquals(204, response.status_code)
response = client.get(url)
self.assertEquals(200, response.status_code)
in_test = "test" in response.content.lower()
in_control = "control" in response.content.lower()
self.assertTrue(in_test != in_control)
found_control = found_control or in_control
found_test = found_test or in_test
self.assertTrue(found_control)
self.assertTrue(found_test)
def testIllegalGroupName(self):
self.doRenderExperiment("username", "foo group",
expect_render_exception=True)
def testBadSyntax(self):
self.doRenderExperimentToken(
"username", ("experiment", self.experiment.name, "group", "yeah"),
expect_parse_exception=True)
def doRenderExperimentToken(self, username, token_tuple,
expect_parse_exception=False,
expect_render_exception=False):
internal_render_result = "rendered"
parser = self.mox.CreateMockAnything()
child_node_list = self.mox.CreateMockAnything()
context = {}
user_factory = self.mox.CreateMockAnything()
token = self.mox.CreateMockAnything()
token.split_contents().AndReturn(token_tuple)
parser.parse(('endexperiment', )).MultipleTimes().AndReturn(
child_node_list)
parser.delete_first_token().MultipleTimes()
user_factory.create_user(context).MultipleTimes().AndReturn(
TestUser(username=username))
child_node_list.render(context).MultipleTimes().AndReturn(
internal_render_result)
self.mox.ReplayAll()
# HACK: there is no way to make a call optional
child_node_list.render(context)
parser.parse(('endexperiment', ))
parser.delete_first_token()
user_factory.create_user(context)
do_parse = lambda: experiment(parser, token, user_factory=user_factory)
if expect_parse_exception:
self.assertRaises(Exception, do_parse)
else:
node = do_parse()
if expect_parse_exception:
self.mox.VerifyAll()
return None
do_render = lambda: node.render(context)
if expect_render_exception:
self.assertRaises(Exception, do_render)
else:
actual_render_result = do_render()
if expect_render_exception:
self.mox.VerifyAll()
return None
self.mox.VerifyAll()
self.assertTrue(actual_render_result == "" or
actual_render_result == internal_render_result)
in_group = (actual_render_result == internal_render_result)
return in_group
def doRenderExperiment(self, username, groupname, **kwargs):
return self.doRenderExperimentToken(
username, ("experiment", self.experiment.name, groupname),
**kwargs)
def testExperimentTag(self):
found_test = False
found_control = False
for i in range(100):
username = "******" % i
in_test = self.doRenderExperiment(username, "test")
found_test = found_test or in_test
found_control = found_control or (not in_test)
# make sure we get the same result from a second call
self.assertEquals(in_test,
self.doRenderExperiment(username, "test"))
# make sure that the result of calling control is the opposite of
# the result of calling test
in_control = self.doRenderExperiment(username, "control")
self.assertEquals(not in_test, in_control)
self.assertTrue(found_control and found_test)
def doRenderClientSideExperiment(self, context, username, experiment_name):
parser = self.mox.CreateMockAnything()
user_factory = self.mox.CreateMockAnything()
token = self.mox.CreateMockAnything()
token.split_contents().AndReturn(("clientsideexperiment",
experiment_name))
user_factory.create_user(context).MultipleTimes().AndReturn(
TestUser(username=username))
self.mox.ReplayAll()
# HACK this is the only way to make a call optional
user_factory.create_user(context)
node = clientsideexperiment(parser, token, user_factory=user_factory)
node.render(context)
self.mox.VerifyAll()
def testClientSideExperimentTag(self):
for i in range(100):
user = User(username="******" % i, email="*****@*****.**" % i)
user.save()
# create a context for a single "request"
c = Context({"user": user})
self.doRenderClientSideExperiment(c, "user%i" % i,
self.experiment.name)
self.assertTrue("client_side_experiments" in c)
self.doRenderClientSideExperiment(c, "user%i" % i,
self.experiment.name)
self.doRenderClientSideExperiment(c, "user%i" % i,
self.other_experiment.name)
self.doRenderClientSideExperiment(c, "user%i" % i,
self.other_experiment.name)
# determine what group the user was assigned to in each experiment
experiment_partcipant = Participant.objects.get(
user=user,
experiment=self.experiment)
other_experiment_partcipant = Participant.objects.get(
user=user,
experiment=self.other_experiment)
# assert that exactly two entries were populated in the
# 'client_side_experiments' dict in the context and that they are
# equal to the names of our two experiments
self.assertEquals(2, len(c['client_side_experiments'].keys()))
self.assertTrue(self.experiment.name in
c['client_side_experiments'])
self.assertTrue(self.other_experiment.name in
c['client_side_experiments'])
# get the values that were put in the client_side_experiments dict
group_name = c['client_side_experiments'][self.experiment.name]
other_group_name = \
c['client_side_experiments'][self.other_experiment.name]
# make sure that the values correctly represent the user's
# enrollments
group_id = experiment_partcipant.group
other_group_id = other_experiment_partcipant.group
self.assertEqual(group_id == Participant.TEST_GROUP,
group_name == "test")
self.assertEqual(group_id == Participant.CONTROL_GROUP,
group_name == "control")
self.assertEqual(
other_group_id == Participant.TEST_GROUP,
other_group_name == "test")
self.assertEqual(
other_group_id == Participant.CONTROL_GROUP,
other_group_name == "control")
def testVerifyHuman(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
other_experiment = Experiment(name="other_experiment")
other_experiment.save()
other_experiment.state = Experiment.ENABLED_STATE
other_experiment.save()
self.client = Client()
original_participants_count = Participant.objects.all().count()
original_anonymous_visitors_count = AnonymousVisitor.objects.all().count()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
response = self.client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(original_participants_count,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count,
AnonymousVisitor.objects.all().count())
confirm_human_url = reverse('experiments.views.confirm_human')
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count+1,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
# calling it again does nothing
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count+1,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
other_experiment_url = reverse(
'experiments.tests.views.experiment_test',
args=[other_experiment.name])
response = self.client.get(other_experiment_url)
self.assertEquals(response.status_code, 200)
# a new participant is immediately created for the new experiment
self.assertEquals(original_participants_count + 2,
Participant.objects.all().count())
# the existing anonymous visitor is reused
self.assertEquals(original_anonymous_visitors_count + 1,
AnonymousVisitor.objects.all().count())
# the new experiment does not cause storage of a temporary enrollment
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count+2,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
class TestDailyReports(TestCase):
def setUp(self):
self.experiment = Experiment(name="test_experiment")
self.experiment.save()
self.experiment.state = Experiment.ENABLED_STATE
self.experiment.save()
self.experiment.start_date = (self.experiment.start_date -
timedelta(days=5))
self.experiment.save()
anonymous_visitor = AnonymousVisitor()
anonymous_visitor.save()
anonymous_participant = Participant(anonymous_visitor=anonymous_visitor,
experiment=self.experiment,
group=Participant.TEST_GROUP)
anonymous_participant.save()
anonymous_participant.enrollment_date = self.experiment.start_date
anonymous_participant.save()
self.other_experiment = Experiment(name="test_experiment2")
self.other_experiment.save()
self.other_experiment.state = Experiment.DISABLED_STATE
self.other_experiment.save()
self.other_experiment.start_date = (date.today() -
timedelta(days=7))
self.other_experiment.end_date = (date.today() -
timedelta(days=3))
self.other_experiment.save()
def testDailyEngagementReport(self):
users_test = []
users_control = []
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
#create users
for i in range(5):
users_control.append(create_user_in_group(self.experiment, i,
Participant.CONTROL_GROUP,
date.today() - timedelta(days=i)))
users_test.append(create_user_in_group(self.experiment, i,
Participant.TEST_GROUP,
date.today() - timedelta(days=i)))
# users_<test|control>[0] were enrolled today, [1] 1 day ago, etc.
report_date = date.today() - timedelta(days=1)
expected_engagement_score_calls = {
(users_test[1], date.today() - timedelta(days=1), report_date): 3.2,
(users_test[2], date.today() - timedelta(days=2), report_date): 2.5,
(users_test[3], date.today() - timedelta(days=3), report_date): 4.1,
(users_test[4], date.today() - timedelta(days=4), report_date): 0,
(users_control[1], date.today() - timedelta(days=1), report_date): 0,
(users_control[2], date.today() - timedelta(days=2), report_date): 0,
(users_control[3], date.today() - timedelta(days=3), report_date): 0,
(users_control[4], date.today() - timedelta(days=4), report_date): 0}
test_case = self
class EngagementScoreCalculatorStub(object):
def calculate_user_engagement_score(self, user,
start_date, end_date):
test_case.assertNotEquals(user, None)
test_case.assertTrue(expected_engagement_score_calls.
has_key((user, start_date, end_date)))
return expected_engagement_score_calls[(user,
start_date, end_date)]
(EngagementReportGenerator(EngagementScoreCalculatorStub()).
generate_daily_report_for_experiment(self.experiment, report_date))
experiment_report = DailyEngagementReport.objects.get(
experiment=self.experiment, date=report_date)
self.assertAlmostEqual((3.2 + 2.5 + 4.1 + 0)/4.0,
experiment_report.test_score)
self.assertAlmostEqual(0.0, experiment_report.control_score)
self.assertEquals(4, experiment_report.test_group_size)
self.assertEquals(4, experiment_report.control_group_size)
self.assertAlmostEqual(96.819293337188498, experiment_report.confidence)
def testZeroParticipantExperiment(self):
mocker = mox.Mox()
engagement_calculator = mocker.CreateMockAnything()
mocker.ReplayAll()
report_date = date.today()
EngagementReportGenerator(engagement_score_calculator=engagement_calculator).generate_daily_report_for_experiment(
self.other_experiment, report_date)
experiment_report = DailyEngagementReport.objects.get(
experiment=self.other_experiment, date=report_date)
mocker.VerifyAll()
self.assertEquals(None, experiment_report.test_score)
self.assertEquals(None, experiment_report.control_score)
self.assertEquals(0, experiment_report.test_group_size)
self.assertEquals(0, experiment_report.control_group_size)
def testGenerateAllDailyEngagementReports(self):
class DummyEngagementCalculator(object):
def calculate_user_engagement_score(self, user, start_date, end_date):
return 7
engagement_report_generator = EngagementReportGenerator(engagement_score_calculator=DummyEngagementCalculator())
engagement_report_generator.generate_daily_report_for_experiment(
self.experiment, date.today() - timedelta(days=2))
engagement_report_generator.generate_daily_report_for_experiment(
self.experiment, date.today() - timedelta(days=3))
engagement_report_generator.generate_all_daily_reports()
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() - timedelta(days=1))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() - timedelta(days=2))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() - timedelta(days=3))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() - timedelta(days=4))
DailyEngagementReport.objects.get(experiment=self.experiment,
date=date.today() - timedelta(days=5))
self.assertEquals(5, DailyEngagementReport.objects.filter(
experiment=self.experiment).count())
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() - timedelta(days=3))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() - timedelta(days=4))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() - timedelta(days=5))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() - timedelta(days=6))
DailyEngagementReport.objects.get(experiment=self.other_experiment,
date=date.today() - timedelta(days=7))
self.assertEquals(5, DailyEngagementReport.objects.filter(
experiment=self.other_experiment).count())
def create_goal_record(self, record_datetime, anonymous_visitor, goal_type):
record = GoalRecord.objects.create(anonymous_visitor=anonymous_visitor,
goal_type=goal_type)
record.created = record_datetime
record.save()
def create_participant(self, anonymous_visitor, experiment, enrollment_date, group):
participant = Participant.objects.create(anonymous_visitor=anonymous_visitor,
experiment=experiment,
group=group)
participant.enrollment_date=enrollment_date
participant.save()
return participant
def testParticipantConversionCalculator(self):
goal_types = [GoalType.objects.create(name=str(i)) for i in range(3)]
anonymous_visitor = AnonymousVisitor.objects.create()
participant = self.create_participant(
anonymous_visitor=anonymous_visitor,
experiment=self.experiment,
enrollment_date=self.experiment.start_date + timedelta(days=2),
group=Participant.TEST_GROUP)
days = [datetime.combine(self.experiment.start_date + timedelta(days=i), time(hour=12))
for i in range(5)]
nb_goal_records = GoalRecord.objects.all().count()
self.create_goal_record(days[0], anonymous_visitor, goal_types[0])
self.create_goal_record(days[0], anonymous_visitor, goal_types[1])
self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
self.create_goal_record(days[1], anonymous_visitor, goal_types[0])
self.create_goal_record(days[2], anonymous_visitor, goal_types[1])
self.create_goal_record(days[3], anonymous_visitor, goal_types[0])
self.create_goal_record(days[4], anonymous_visitor, goal_types[0])
self.create_goal_record(days[4], anonymous_visitor, goal_types[0])
self.assertEquals(nb_goal_records + 8, GoalRecord.objects.all().count())
# wasn't enrolled yet!
self.assertEquals(calculate_participant_conversion(participant, goal_types[0], days[0]), 0)
self.assertEquals(calculate_participant_conversion(participant, goal_types[1], days[0]), 0)
self.assertEquals(calculate_participant_conversion(participant, goal_types[2], days[0]), 0)
self.assertEquals(calculate_participant_conversion(participant, None, days[0]), 0)
self.assertEquals(calculate_participant_conversion(participant, goal_types[0], days[1]), 0)
self.assertEquals(calculate_participant_conversion(participant, goal_types[1], days[1]), 0)
self.assertEquals(calculate_participant_conversion(participant, goal_types[2], days[1]), 0)
self.assertEquals(calculate_participant_conversion(participant, None, days[1]), 0)
# now enrolled
self.assertEquals(calculate_participant_conversion(participant, goal_types[0], days[2]), 0)
self.assertEquals(calculate_participant_conversion(participant, goal_types[1], days[2]), 1)
self.assertEquals(calculate_participant_conversion(participant, goal_types[2], days[2]), 0)
# "any" is one
self.assertEquals(calculate_participant_conversion(participant, None, days[2]), 1)
self.assertEquals(calculate_participant_conversion(participant, goal_types[0], days[3]), 1)
self.assertEquals(calculate_participant_conversion(participant, goal_types[1], days[3]), 1)
self.assertEquals(calculate_participant_conversion(participant, goal_types[2], days[3]), 0)
# "any" is one, even though two different goals were achieved
self.assertEquals(calculate_participant_conversion(participant, None, days[3]), 1)
# there were three conversions on this day for goal 0, but we only count the first!
self.assertEquals(calculate_participant_conversion(participant, goal_types[0], days[4]), 1)
self.assertEquals(calculate_participant_conversion(participant, goal_types[1], days[4]), 1)
self.assertEquals(calculate_participant_conversion(participant, goal_types[2], days[4]), 0)
self.assertEquals(calculate_participant_conversion(participant, None, days[4]), 1)
def testGoalTypeConversionCalculator(self):
mocker = mox.Mox()
participants = [mocker.CreateMockAnything(),
mocker.CreateMockAnything(),
mocker.CreateMockAnything()]
goal_type = mocker.CreateMockAnything()
report_date = mocker.CreateMockAnything()
participant_conversion_calculator = mocker.CreateMockAnything()
participant_conversion_calculator(
participants[0], goal_type, report_date).InAnyOrder().AndReturn(1)
participant_conversion_calculator(
participants[1], goal_type, report_date).InAnyOrder().AndReturn(0)
participant_conversion_calculator(
participants[2], goal_type, report_date).InAnyOrder().AndReturn(1)
mocker.ReplayAll()
self.assertEquals(2, calculate_goal_type_conversion(
goal_type, participants, report_date, participant_conversion_calculator))
mocker.VerifyAll()
def testExperimentGroupParticipantFinder(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)]
experiment_participant_groups = [
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEquals(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEquals(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEquals(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEquals(2, len(ex2day5testvisitors))
def testGetConversionData(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
yesterday = date.today() - timedelta(days=1)
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = yesterday
experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
report = DailyConversionReport.objects.create(experiment=experiment,
date=yesterday,
overall_test_conversion=17,
overall_control_conversion=12,
test_group_size=139,
control_group_size=142,
confidence=87.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[0],
test_conversion=11,
control_conversion=0,
confidence=65.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[1],
test_conversion=0,
control_conversion=21,
confidence=None)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[2],
test_conversion=23,
control_conversion=21,
confidence=100)
data = get_conversion_data(experiment, yesterday)
self.assertEquals(data['date'], yesterday)
self.assertTrue("totals" in data)
self.assertTrue("goal_types" in data)
self.assertEquals(4, len(data["goal_types"].keys()))
for goal_type in goal_types[0:3]:
self.assertTrue(goal_type.name in data["goal_types"])
goal_type_data = data["goal_types"][goal_type.name]
self.assertTrue("test_count" in goal_type_data)
self.assertTrue("control_count" in goal_type_data)
self.assertTrue("test_rate" in goal_type_data)
self.assertTrue("control_rate" in goal_type_data)
self.assertTrue("improvement" in goal_type_data)
self.assertTrue("confidence" in goal_type_data)
self.assertEquals(None, data["goal_types"][goal_types[3].name])
self.assertEquals(139, data["test_group_size"])
self.assertEquals(142, data["control_group_size"])
totals = data["totals"]
expected_test_rate = 17. / 139. * 100.
expected_control_rate = 12. / 142. * 100.
expected_improvement = (expected_test_rate - expected_control_rate) / expected_control_rate * 100.
self.assertAlmostEquals(expected_test_rate, totals["test_rate"])
self.assertAlmostEquals(expected_control_rate, totals["control_rate"])
self.assertAlmostEquals(expected_improvement, totals["improvement"])
self.assertAlmostEquals(87.3, totals["confidence"])
self.assertEquals(17, totals["test_count"])
self.assertEquals(12, totals["control_count"])
self.assertEquals(0, data["goal_types"][goal_types[0].name]["control_rate"])
self.assertAlmostEquals(11./139*100., data["goal_types"][goal_types[0].name]["test_rate"])
self.assertEquals(None, data["goal_types"][goal_types[0].name]["improvement"])
self.assertAlmostEquals(65.3, data["goal_types"][goal_types[0].name]["confidence"])
self.assertEquals(11, data["goal_types"][goal_types[0].name]["test_count"])
self.assertEquals(0, data["goal_types"][goal_types[0].name]["control_count"])
self.assertAlmostEquals(21./142*100., data["goal_types"][goal_types[1].name]["control_rate"])
self.assertEquals(None, data["goal_types"][goal_types[1].name]["confidence"])
self.assertEquals(None, data["goal_types"][goal_types[1].name]["improvement"])
self.assertAlmostEquals((23./139-21./142)/(21./142)*100.,
data["goal_types"][goal_types[2].name]["improvement"])
#TODO test with zero participants and check rate == None
#TODO sometimes confidence cannot be calculated and must return None. Add a test to verify this.
def testConversionReportGenerator(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)])
calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEquals(results.count(), 5)
report_days = [ d.date() for d in days[2:7]]
for i in range(5):
self.assertEquals(results[i].date, report_days[4-i])
# Day 2
self.assertEquals(12, results[4].test_group_size)
self.assertEquals(7, results[4].control_group_size)
self.assertEquals(5, results[4].overall_test_conversion)
self.assertEquals(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[0])[0].test_conversion
self.assertEquals(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[2])[0].control_conversion
self.assertEquals(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEquals(5, results[3].test_group_size)
# Day 4
self.assertEquals(0, results[2].test_group_size)
self.assertEquals(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2],
goal_type=goal_types[0])[0].confidence
self.assertEquals(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1],
goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testGetConversionData(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
yesterday = date.today() - timedelta(days=1)
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = yesterday
experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
report = DailyConversionReport.objects.create(
experiment=experiment,
date=yesterday,
overall_test_conversion=17,
overall_control_conversion=12,
test_group_size=139,
control_group_size=142,
confidence=87.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[0],
test_conversion=11,
control_conversion=0,
confidence=65.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[1],
test_conversion=0,
control_conversion=21,
confidence=None)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[2],
test_conversion=23,
control_conversion=21,
confidence=100)
data = get_conversion_data(experiment, yesterday)
self.assertEquals(data['date'], yesterday)
self.assertTrue("totals" in data)
self.assertTrue("goal_types" in data)
self.assertEquals(4, len(data["goal_types"].keys()))
for goal_type in goal_types[0:3]:
self.assertTrue(goal_type.name in data["goal_types"])
goal_type_data = data["goal_types"][goal_type.name]
self.assertTrue("test_count" in goal_type_data)
self.assertTrue("control_count" in goal_type_data)
self.assertTrue("test_rate" in goal_type_data)
self.assertTrue("control_rate" in goal_type_data)
self.assertTrue("improvement" in goal_type_data)
self.assertTrue("confidence" in goal_type_data)
self.assertEquals(None, data["goal_types"][goal_types[3].name])
self.assertEquals(139, data["test_group_size"])
self.assertEquals(142, data["control_group_size"])
totals = data["totals"]
expected_test_rate = 17. / 139. * 100.
expected_control_rate = 12. / 142. * 100.
expected_improvement = (expected_test_rate - expected_control_rate
) / expected_control_rate * 100.
self.assertAlmostEquals(expected_test_rate, totals["test_rate"])
self.assertAlmostEquals(expected_control_rate, totals["control_rate"])
self.assertAlmostEquals(expected_improvement, totals["improvement"])
self.assertAlmostEquals(87.3, totals["confidence"])
self.assertEquals(17, totals["test_count"])
self.assertEquals(12, totals["control_count"])
self.assertEquals(
0, data["goal_types"][goal_types[0].name]["control_rate"])
self.assertAlmostEquals(
11. / 139 * 100.,
data["goal_types"][goal_types[0].name]["test_rate"])
self.assertEquals(
None, data["goal_types"][goal_types[0].name]["improvement"])
self.assertAlmostEquals(
65.3, data["goal_types"][goal_types[0].name]["confidence"])
self.assertEquals(11,
data["goal_types"][goal_types[0].name]["test_count"])
self.assertEquals(
0, data["goal_types"][goal_types[0].name]["control_count"])
self.assertAlmostEquals(
21. / 142 * 100.,
data["goal_types"][goal_types[1].name]["control_rate"])
self.assertEquals(None,
data["goal_types"][goal_types[1].name]["confidence"])
self.assertEquals(
None, data["goal_types"][goal_types[1].name]["improvement"])
self.assertAlmostEquals(
(23. / 139 - 21. / 142) / (21. / 142) * 100.,
data["goal_types"][goal_types[2].name]["improvement"])