def testAnonymousUser(self):
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
for i in range(100):
user = TestUser()
in_test = Experiment.test("enabled", user)
anonymous_id = user.get_anonymous_id()
self.assertNotEquals(None, anonymous_id)
in_control = Experiment.control("enabled", user)
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
self.assertEquals(in_control, Experiment.control("enabled", user))
self.assertEquals(user.get_anonymous_id(), anonymous_id)
if in_test:
test_user = user
if in_control:
control_user = user
self.assertTrue(test_user and control_user)
user = TestUser()
experiment = Experiment(name="disabled")
experiment.save()
self.assertFalse(Experiment.test("disabled", user))
self.assertEquals(None, user.get_anonymous_id())
def testAnonymousUser(self):
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
for i in range(100):
user = TestUser()
in_test = Experiment.test("enabled", user)
anonymous_id = user.get_anonymous_id()
self.assertNotEqual(None, anonymous_id)
in_control = Experiment.control("enabled", user)
self.assertEqual(user.get_anonymous_id(), anonymous_id)
self.assertNotEqual(in_test, in_control)
self.assertEqual(in_test, Experiment.test("enabled", user))
self.assertEqual(user.get_anonymous_id(), anonymous_id)
self.assertEqual(in_control, Experiment.control("enabled", user))
self.assertEqual(user.get_anonymous_id(), anonymous_id)
if in_test:
test_user = user
if in_control:
control_user = user
self.assertTrue(test_user and control_user)
user = TestUser()
experiment = Experiment(name="disabled")
experiment.save()
self.assertFalse(Experiment.test("disabled", user))
self.assertEqual(None, user.get_anonymous_id())
def testPromotedExperiment(self):
# promoted test (unenrolled user)
experiment = Experiment(name="promoted")
experiment.save()
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
user = TestUser(username="******")
self.assertTrue(Experiment.test("promoted", user))
self.assertFalse(Experiment.control("promoted", user))
def testPromotedExperiment(self):
# promoted test (unenrolled user)
experiment = Experiment(name="promoted")
experiment.save()
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
user = TestUser(username="******")
self.assertTrue(Experiment.test("promoted", user))
self.assertFalse(Experiment.control("promoted", user))
def testEnabledPromotedAndDisabledExperiment(self):
# enabled test, new user (prove we get both results)
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
test_user = None
control_user = None
for i in range(100):
user = TestUser(username="******" % i)
in_test = Experiment.test("enabled", user)
in_control = Experiment.control("enabled", user)
self.assertNotEqual(in_test, in_control)
self.assertEqual(in_test, Experiment.test("enabled", user))
self.assertEqual(in_control, Experiment.control("enabled", user))
if in_test:
test_user = user
if in_control:
control_user = user
self.assertNotEqual(None, test_user)
self.assertNotEqual(None, control_user)
self.assertNotEqual(test_user, control_user)
# promoted test with an enrolled user
# (prove we get both results, then that all users are in test)
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
self.assertFalse(Experiment.control("enabled", control_user))
self.assertFalse(Experiment.control("enabled", test_user))
self.assertTrue(Experiment.test("enabled", control_user))
self.assertTrue(Experiment.test("enabled", test_user))
# disabled test with an enrolled user
# (prove we get both results, then that all users are in control)
experiment.state = Experiment.DISABLED_STATE
experiment.save()
self.assertTrue(Experiment.control("enabled", control_user))
self.assertTrue(Experiment.control("enabled", test_user))
self.assertFalse(Experiment.test("enabled", control_user))
self.assertFalse(Experiment.test("enabled", test_user))
def testEnabledPromotedAndDisabledExperiment(self):
# enabled test, new user (prove we get both results)
experiment = Experiment(name="enabled")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
test_user = None
control_user = None
for i in range(100):
user = TestUser(username="******" % i)
in_test = Experiment.test("enabled", user)
in_control = Experiment.control("enabled", user)
self.assertNotEquals(in_test, in_control)
self.assertEquals(in_test, Experiment.test("enabled", user))
self.assertEquals(in_control, Experiment.control("enabled", user))
if in_test:
test_user = user
if in_control:
control_user = user
self.assertNotEquals(None, test_user)
self.assertNotEquals(None, control_user)
self.assertNotEquals(test_user, control_user)
# promoted test with an enrolled user
# (prove we get both results, then that all users are in test)
experiment.state = Experiment.PROMOTED_STATE
experiment.save()
self.assertFalse(Experiment.control("enabled", control_user))
self.assertFalse(Experiment.control("enabled", test_user))
self.assertTrue(Experiment.test("enabled", control_user))
self.assertTrue(Experiment.test("enabled", test_user))
# disabled test with an enrolled user
# (prove we get both results, then that all users are in control)
experiment.state = Experiment.DISABLED_STATE
experiment.save()
self.assertTrue(Experiment.control("enabled", control_user))
self.assertTrue(Experiment.control("enabled", test_user))
self.assertFalse(Experiment.test("enabled", control_user))
self.assertFalse(Experiment.test("enabled", test_user))
def testParticipantEnrollment(self):
experiment1 = Experiment(name="Experiment 1")
experiment1.save()
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
for i in range(1000):
username = "******" % i
in_test1 = Experiment.test(experiment1.name,
TestUser(username=username))
self.assertEquals(
in_test1, not Experiment.control(experiment1.name,
TestUser(username=username)))
if in_test1:
num_test1 += 1
else:
num_control1 += 1
in_test2 = not Experiment.control(experiment2.name,
TestUser(username=username))
self.assertEquals(
in_test2,
Experiment.test(experiment2.name, TestUser(username=username)))
if in_test2:
num_test2 += 1
else:
num_control2 += 1
self.assert_(num_control1 > 400)
self.assert_(num_control2 > 400)
self.assert_(num_test1 > 400)
self.assert_(num_test2 > 400)
def testGroupSanity(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
for i in range(100):
self.client = Client()
response = self.client.get(experiment_url)
self.assertEqual(response.status_code, 200)
self.assertTrue(response.content.strip().lower() in ("test",
"control"))
def testGroupSanity(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
for i in range(100):
self.client = Client()
response = self.client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertTrue(response.content.strip().lower() in ("test",
"control"))
def testParticipantEnrollment(self):
experiment1 = Experiment(name="Experiment 1")
experiment1.save()
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
num_control1 = 0
num_test1 = 0
num_control2 = 0
num_test2 = 0
for i in range(1000):
username="******" % i
in_test1 = Experiment.test(experiment1.name, TestUser(username=username))
self.assertEquals(in_test1, not Experiment.control(experiment1.name,
TestUser(username=username)))
if in_test1:
num_test1 += 1
else:
num_control1 += 1
in_test2 = not Experiment.control(experiment2.name, TestUser(username=username))
self.assertEquals(in_test2, Experiment.test(experiment2.name,
TestUser(username=username)))
if in_test2:
num_test2 += 1
else:
num_control2 += 1
self.assert_(num_control1 > 400)
self.assert_(num_control2 > 400)
self.assert_(num_test1 > 400)
self.assert_(num_test2 > 400)
def testBotExclusion(self):
experiment = Experiment(name="bot_experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
user = TestUser(verified_human=False)
participants_count = Participant.objects.all().count()
in_test = Experiment.test(experiment.name, user)
self.assertEquals(None, user.get_anonymous_id())
self.assertEquals(participants_count, Participant.objects.all().count())
enrollments = user.get_added_enrollments()
self.assertEquals(len(enrollments.keys()), 1)
self.assertTrue(experiment.name in enrollments.keys())
def testBotExclusion(self):
experiment = Experiment(name="bot_experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
user = TestUser(verified_human=False)
participants_count = Participant.objects.all().count()
in_test = Experiment.test(experiment.name, user)
self.assertEquals(None, user.get_anonymous_id())
self.assertEquals(participants_count,
Participant.objects.all().count())
enrollments = user.get_added_enrollments()
self.assertEquals(len(enrollments.keys()), 1)
self.assertTrue(experiment.name in enrollments.keys())
def testExperimentStates(self):
experiment1 = Experiment(name="test_experiment_1")
experiment1.save()
experiment1 = Experiment.objects.get(name="test_experiment_1")
self.assertRaises(Exception,
lambda: Experiment(name="test_experiment_1").save())
#test default values
self.assertEquals(experiment1.state, Experiment.DISABLED_STATE)
self.assertEquals(experiment1.start_date, None)
self.assertEquals(experiment1.end_date, None)
#enable the experiment
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
self.assertNotEquals(experiment1.start_date, None)
self.assertEquals(experiment1.end_date, None)
#disable the experiement
old_start_date = experiment1.start_date
experiment1.state = Experiment.DISABLED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertNotEquals(experiment1.end_date, None)
#enable the experiment
old_end_date = experiment1.end_date
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertEquals(experiment1.end_date, old_end_date)
#promote the experiment
experiment1.state = Experiment.PROMOTED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertEquals(experiment1.end_date, old_end_date)
experiment2 = Experiment(name="Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
self.assertEquals(experiment2.start_date, experiment1.start_date)
experiment2.state = Experiment.PROMOTED_STATE
experiment2.save()
self.assertNotEquals(experiment2.start_date, None)
self.assertNotEquals(experiment2.end_date, None)
def testExperimentStates(self):
experiment1 = Experiment(name="test_experiment_1")
experiment1.save()
experiment1 = Experiment.objects.get(name="test_experiment_1")
self.assertRaises(Exception,
lambda: Experiment(name="test_experiment_1").save())
#test default values
self.assertEquals(experiment1.state, Experiment.DISABLED_STATE)
self.assertEquals(experiment1.start_date, None)
self.assertEquals(experiment1.end_date, None)
#enable the experiment
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
self.assertNotEquals(experiment1.start_date, None)
self.assertEquals(experiment1.end_date, None)
#disable the experiement
old_start_date = experiment1.start_date
experiment1.state = Experiment.DISABLED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertNotEquals(experiment1.end_date, None)
#enable the experiment
old_end_date = experiment1.end_date
experiment1.state = Experiment.ENABLED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertEquals(experiment1.end_date, old_end_date)
#promote the experiment
experiment1.state = Experiment.PROMOTED_STATE
experiment1.save()
self.assertEquals(experiment1.start_date, old_start_date)
self.assertEquals(experiment1.end_date, old_end_date)
experiment2 = Experiment(name= "Experiment 2")
experiment2.save()
experiment2.state = Experiment.ENABLED_STATE
experiment2.save()
self.assertEquals(experiment2.start_date, experiment1.start_date)
experiment2.state = Experiment.PROMOTED_STATE
experiment2.save()
self.assertNotEquals(experiment2.start_date, None)
self.assertNotEquals(experiment2.end_date, None)
def testExperimentGroupParticipantFinder(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [AnonymousVisitor.objects.create() for i in range(10)]
experiment_participant_groups = [
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEqual(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEqual(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEqual(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEqual(2, len(ex2day5testvisitors))
def testPngResponse(self):
experiment = Experiment(name="test-experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
goal_type = GoalType(name='test-goal')
goal_type.save()
experiment_url = reverse(
"django_lean.experiments.tests.views.experiment_test",
args=[experiment.name])
confirm_human_url = reverse(
"django_lean.experiments.views.confirm_human")
client = Client()
# we can call with invalid or inexisting names, the response is the same
self.call_goal(client, '')
self.call_goal(client, 'unknown-goal')
# this is an anonymous visitor not enrolled in an experiment,
# so no records should be created
self.call_goal(client, goal_type.name)
self.assertEquals(
0,
GoalRecord.objects.filter(goal_type=goal_type).count())
nb_anonymous_visitors = AnonymousVisitor.objects.count()
# force the user to be a verified human
response = client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
# force the anonymous visitor to be enrolled in an experiment
response = client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
client.get('/test-experiment/%s' % experiment.name)
self.assertEquals(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
# now call an existing goal again - it should be recorded
self.call_goal(client, goal_type.name)
self.assertEquals(
1,
GoalRecord.objects.filter(goal_type=goal_type).count())
# should be recorded again
self.call_goal(client, goal_type.name)
self.assertEquals(
2,
GoalRecord.objects.filter(goal_type=goal_type).count())
# validate that both of the records have the same anonymous_visitor
two_goal_records = GoalRecord.objects.filter(goal_type=goal_type)
self.assertEquals(two_goal_records[0].anonymous_visitor,
two_goal_records[1].anonymous_visitor)
# try it with a registered user
client = Client()
user = User(username="******", email="*****@*****.**")
user.set_password("password")
user.save()
response = client.login(username=user.username, password='******')
self.assertTrue(response)
# force the registered user to be enrolled in an experiment
client.get('/test-experiment/%s' % experiment.name)
self.call_goal(client, goal_type.name)
# since the user was registered, no new records should be created
self.assertEquals(
2,
GoalRecord.objects.filter(goal_type=goal_type).count())
def testVerifyHuman(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
other_experiment = Experiment(name="other_experiment")
other_experiment.save()
other_experiment.state = Experiment.ENABLED_STATE
other_experiment.save()
self.client = Client()
original_participants_count = Participant.objects.all().count()
original_anonymous_visitors_count = AnonymousVisitor.objects.all(
).count()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
response = self.client.get(experiment_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(original_participants_count,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count,
AnonymousVisitor.objects.all().count())
confirm_human_url = reverse('experiments.views.confirm_human')
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count + 1,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count + 1,
AnonymousVisitor.objects.all().count())
# calling it again does nothing
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count + 1,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count + 1,
AnonymousVisitor.objects.all().count())
other_experiment_url = reverse(
'experiments.tests.views.experiment_test',
args=[other_experiment.name])
response = self.client.get(other_experiment_url)
self.assertEquals(response.status_code, 200)
# a new participant is immediately created for the new experiment
self.assertEquals(original_participants_count + 2,
Participant.objects.all().count())
# the existing anonymous visitor is reused
self.assertEquals(original_anonymous_visitors_count + 1,
AnonymousVisitor.objects.all().count())
# the new experiment does not cause storage of a temporary enrollment
response = self.client.get(confirm_human_url)
self.assertEquals(response.status_code, 204)
self.assertEquals(0, len(response.content))
self.assertEquals(original_participants_count + 2,
Participant.objects.all().count())
self.assertEquals(original_anonymous_visitors_count + 1,
AnonymousVisitor.objects.all().count())
def testVerifyHuman(self):
experiment = Experiment(name="experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
other_experiment = Experiment(name="other_experiment")
other_experiment.save()
other_experiment.state = Experiment.ENABLED_STATE
other_experiment.save()
self.client = Client()
original_participants_count = Participant.objects.all().count()
original_anonymous_visitors_count = AnonymousVisitor.objects.all().count()
experiment_url = reverse('experiments.tests.views.experiment_test',
args=[experiment.name])
response = self.client.get(experiment_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(original_participants_count,
Participant.objects.all().count())
self.assertEqual(original_anonymous_visitors_count,
AnonymousVisitor.objects.all().count())
confirm_human_url = reverse('experiments.views.confirm_human')
response = self.client.get(confirm_human_url)
self.assertEqual(response.status_code, 204)
self.assertEqual(0, len(response.content))
self.assertEqual(original_participants_count+1,
Participant.objects.all().count())
self.assertEqual(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
# calling it again does nothing
response = self.client.get(confirm_human_url)
self.assertEqual(response.status_code, 204)
self.assertEqual(0, len(response.content))
self.assertEqual(original_participants_count+1,
Participant.objects.all().count())
self.assertEqual(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
other_experiment_url = reverse(
'experiments.tests.views.experiment_test',
args=[other_experiment.name])
response = self.client.get(other_experiment_url)
self.assertEqual(response.status_code, 200)
# a new participant is immediately created for the new experiment
self.assertEqual(original_participants_count + 2,
Participant.objects.all().count())
# the existing anonymous visitor is reused
self.assertEqual(original_anonymous_visitors_count + 1,
AnonymousVisitor.objects.all().count())
# the new experiment does not cause storage of a temporary enrollment
response = self.client.get(confirm_human_url)
self.assertEqual(response.status_code, 204)
self.assertEqual(0, len(response.content))
self.assertEqual(original_participants_count+2,
Participant.objects.all().count())
self.assertEqual(original_anonymous_visitors_count+1,
AnonymousVisitor.objects.all().count())
def testExperimentGroupParticipantFinder(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
anonymous_visitors = [
AnonymousVisitor.objects.create() for i in range(10)
]
experiment_participant_groups = [
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=experiment,
enrollment_date=days[2],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[1],
experiment=experiment,
enrollment_date=days[2],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[3],
experiment=experiment,
enrollment_date=days[3],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[4],
experiment=experiment,
enrollment_date=days[4],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[6],
experiment=experiment,
enrollment_date=days[6],
group=Participant.TEST_GROUP)
],
[
self.create_participant(
anonymous_visitor=anonymous_visitors[0],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.CONTROL_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[2],
experiment=other_experiment,
enrollment_date=days[0],
group=Participant.TEST_GROUP),
self.create_participant(
anonymous_visitor=anonymous_visitors[5],
experiment=other_experiment,
enrollment_date=days[4],
group=Participant.TEST_GROUP)
]
]
ex1day2 = find_experiment_group_participants(Participant.TEST_GROUP,
experiment, days[2])
ex1day2visitors = [p.anonymous_visitor for p in ex1day2]
self.assertTrue(anonymous_visitors[0] in ex1day2visitors)
self.assertEquals(1, len(ex1day2visitors))
ex1day4test = find_experiment_group_participants(
Participant.TEST_GROUP, experiment, days[4])
ex1day4testvisitors = [p.anonymous_visitor for p in ex1day4test]
self.assertTrue(anonymous_visitors[0] in ex1day4testvisitors)
self.assertTrue(anonymous_visitors[3] in ex1day4testvisitors)
self.assertEquals(2, len(ex1day4testvisitors))
ex1day4control = find_experiment_group_participants(
Participant.CONTROL_GROUP, experiment, days[4])
ex1day4controlvisitors = [p.anonymous_visitor for p in ex1day4control]
self.assertTrue(anonymous_visitors[1] in ex1day4controlvisitors)
self.assertTrue(anonymous_visitors[4] in ex1day4controlvisitors)
self.assertEquals(2, len(ex1day4controlvisitors))
ex2day5test = find_experiment_group_participants(
Participant.TEST_GROUP, other_experiment, days[5])
ex2day5testvisitors = [p.anonymous_visitor for p in ex2day5test]
self.assertTrue(anonymous_visitors[2] in ex2day5testvisitors)
self.assertTrue(anonymous_visitors[5] in ex2day5testvisitors)
self.assertEquals(2, len(ex2day5testvisitors))
def testGetConversionData(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
yesterday = date.today() - timedelta(days=1)
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = yesterday
experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
report = DailyConversionReport.objects.create(
experiment=experiment,
date=yesterday,
overall_test_conversion=17,
overall_control_conversion=12,
test_group_size=139,
control_group_size=142,
confidence=87.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[0],
test_conversion=11,
control_conversion=0,
confidence=65.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[1],
test_conversion=0,
control_conversion=21,
confidence=None)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[2],
test_conversion=23,
control_conversion=21,
confidence=100)
data = get_conversion_data(experiment, yesterday)
self.assertEquals(data['date'], yesterday)
self.assertTrue("totals" in data)
self.assertTrue("goal_types" in data)
self.assertEquals(4, len(data["goal_types"].keys()))
for goal_type in goal_types[0:3]:
self.assertTrue(goal_type.name in data["goal_types"])
goal_type_data = data["goal_types"][goal_type.name]
self.assertTrue("test_count" in goal_type_data)
self.assertTrue("control_count" in goal_type_data)
self.assertTrue("test_rate" in goal_type_data)
self.assertTrue("control_rate" in goal_type_data)
self.assertTrue("improvement" in goal_type_data)
self.assertTrue("confidence" in goal_type_data)
self.assertEquals(None, data["goal_types"][goal_types[3].name])
self.assertEquals(139, data["test_group_size"])
self.assertEquals(142, data["control_group_size"])
totals = data["totals"]
expected_test_rate = 17. / 139. * 100.
expected_control_rate = 12. / 142. * 100.
expected_improvement = (expected_test_rate - expected_control_rate
) / expected_control_rate * 100.
self.assertAlmostEquals(expected_test_rate, totals["test_rate"])
self.assertAlmostEquals(expected_control_rate, totals["control_rate"])
self.assertAlmostEquals(expected_improvement, totals["improvement"])
self.assertAlmostEquals(87.3, totals["confidence"])
self.assertEquals(17, totals["test_count"])
self.assertEquals(12, totals["control_count"])
self.assertEquals(
0, data["goal_types"][goal_types[0].name]["control_rate"])
self.assertAlmostEquals(
11. / 139 * 100.,
data["goal_types"][goal_types[0].name]["test_rate"])
self.assertEquals(
None, data["goal_types"][goal_types[0].name]["improvement"])
self.assertAlmostEquals(
65.3, data["goal_types"][goal_types[0].name]["confidence"])
self.assertEquals(11,
data["goal_types"][goal_types[0].name]["test_count"])
self.assertEquals(
0, data["goal_types"][goal_types[0].name]["control_count"])
self.assertAlmostEquals(
21. / 142 * 100.,
data["goal_types"][goal_types[1].name]["control_rate"])
self.assertEquals(None,
data["goal_types"][goal_types[1].name]["confidence"])
self.assertEquals(
None, data["goal_types"][goal_types[1].name]["improvement"])
self.assertAlmostEquals(
(23. / 139 - 21. / 142) / (21. / 142) * 100.,
data["goal_types"][goal_types[2].name]["improvement"])
def testConversionReportGenerator(self):
days = [
datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)
]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment,
day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(
data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants,
day.date()).InAnyOrder().AndReturn(
data[group]["conversions"][int(
goal_type.name)])
calculator(None, mock_participants,
day.date()).InAnyOrder().AndReturn(
sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(
calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEquals(results.count(), 5)
report_days = [d.date() for d in days[2:7]]
for i in range(5):
self.assertEquals(results[i].date, report_days[4 - i])
# Day 2
self.assertEquals(12, results[4].test_group_size)
self.assertEquals(7, results[4].control_group_size)
self.assertEquals(5, results[4].overall_test_conversion)
self.assertEquals(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[0])[0].test_conversion
self.assertEquals(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4], goal_type=goal_types[2])[0].control_conversion
self.assertEquals(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEquals(5, results[3].test_group_size)
# Day 4
self.assertEquals(0, results[2].test_group_size)
self.assertEquals(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2], goal_type=goal_types[0])[0].confidence
self.assertEquals(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1], goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029,
day_5_goal_0_confidence,
places=6)
def testGetConversionData(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
yesterday = date.today() - timedelta(days=1)
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = yesterday
experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(4)]
report = DailyConversionReport.objects.create(experiment=experiment,
date=yesterday,
overall_test_conversion=17,
overall_control_conversion=12,
test_group_size=139,
control_group_size=142,
confidence=87.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[0],
test_conversion=11,
control_conversion=0,
confidence=65.3)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[1],
test_conversion=0,
control_conversion=21,
confidence=None)
DailyConversionReportGoalData.objects.create(report=report,
goal_type=goal_types[2],
test_conversion=23,
control_conversion=21,
confidence=100)
data = get_conversion_data(experiment, yesterday)
self.assertEqual(data['date'], yesterday)
self.assertTrue("totals" in data)
self.assertTrue("goal_types" in data)
self.assertEqual(4, len(data["goal_types"].keys()))
for goal_type in goal_types[0:3]:
self.assertTrue(goal_type.name in data["goal_types"])
goal_type_data = data["goal_types"][goal_type.name]
self.assertTrue("test_count" in goal_type_data)
self.assertTrue("control_count" in goal_type_data)
self.assertTrue("test_rate" in goal_type_data)
self.assertTrue("control_rate" in goal_type_data)
self.assertTrue("improvement" in goal_type_data)
self.assertTrue("confidence" in goal_type_data)
self.assertEqual(None, data["goal_types"][goal_types[3].name])
self.assertEqual(139, data["test_group_size"])
self.assertEqual(142, data["control_group_size"])
totals = data["totals"]
expected_test_rate = 17. / 139. * 100.
expected_control_rate = 12. / 142. * 100.
expected_improvement = (expected_test_rate - expected_control_rate) / expected_control_rate * 100.
self.assertAlmostEqual(expected_test_rate, totals["test_rate"])
self.assertAlmostEqual(expected_control_rate, totals["control_rate"])
self.assertAlmostEqual(expected_improvement, totals["improvement"])
self.assertAlmostEqual(87.3, totals["confidence"])
self.assertEqual(17, totals["test_count"])
self.assertEqual(12, totals["control_count"])
self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_rate"])
self.assertAlmostEqual(11./139*100., data["goal_types"][goal_types[0].name]["test_rate"])
self.assertEqual(None, data["goal_types"][goal_types[0].name]["improvement"])
self.assertAlmostEqual(65.3, data["goal_types"][goal_types[0].name]["confidence"])
self.assertEqual(11, data["goal_types"][goal_types[0].name]["test_count"])
self.assertEqual(0, data["goal_types"][goal_types[0].name]["control_count"])
self.assertAlmostEqual(21./142*100., data["goal_types"][goal_types[1].name]["control_rate"])
self.assertEqual(None, data["goal_types"][goal_types[1].name]["confidence"])
self.assertEqual(None, data["goal_types"][goal_types[1].name]["improvement"])
self.assertAlmostEqual((23./139-21./142)/(21./142)*100.,
data["goal_types"][goal_types[2].name]["improvement"])
def testConversionReportGenerator(self):
days = [datetime.combine(date.today() + timedelta(days=i), time(hour=12))
for i in range(-7, 0)]
experiment = Experiment(name="experiment1")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
experiment.start_date = days[2].date()
experiment.save()
other_experiment = Experiment(name="experiment2")
other_experiment.save()
other_experiment.state = Experiment.DISABLED_STATE
other_experiment.save()
other_experiment.start_date = days[0].date()
other_experiment.end_date = days[4].date()
other_experiment.save()
goal_types = [GoalType.objects.create(name="%s" % i) for i in range(3)]
# experiment starts on days[2]
# other experiment starts on days[0]
mocker = mox.Mox()
finder = mocker.CreateMockAnything()
calculator = mocker.CreateMockAnything()
default_data = {
Participant.TEST_GROUP: {
"count": 110,
"conversions": [23, 12, 9]
},
Participant.CONTROL_GROUP: {
"count": 130,
"conversions": [12, 47, 5]
}
}
day_2_data = {
Participant.TEST_GROUP: {
"count": 12,
"conversions": [0, 2, 3]
},
Participant.CONTROL_GROUP: {
"count": 7,
"conversions": [1, 0, 3]
}
}
day_3_data = {
Participant.TEST_GROUP: {
"count": 5,
"conversions": [1, 0, 3]
},
Participant.CONTROL_GROUP: {
"count": 12,
"conversions": [0, 0, 0]
}
}
day_4_data = {
Participant.TEST_GROUP: {
"count": 0,
"conversions": [0, 0, 0]
},
Participant.CONTROL_GROUP: {
"count": 25,
"conversions": [2, 3, 7]
}
}
for day in days[2:7]:
data = default_data
if day == days[2]:
data = day_2_data
if day == days[3]:
data = day_3_data
if day == days[4]:
data = day_4_data
for group in (Participant.TEST_GROUP, Participant.CONTROL_GROUP):
mock_participants = mocker.CreateMockAnything()
finder(group, experiment, day.date()).InAnyOrder().AndReturn(mock_participants)
mock_participants.count().MultipleTimes().AndReturn(data[group]["count"])
for goal_type in goal_types:
calculator(goal_type, mock_participants, day.date()).InAnyOrder().AndReturn(data[group]["conversions"][int(goal_type.name)])
calculator(None, mock_participants, day.date()).InAnyOrder().AndReturn(sum(data[group]["conversions"]))
mocker.ReplayAll()
for d in days[2:7]:
ConversionReportGenerator(calculator, finder).generate_daily_report_for_experiment(
experiment, d.date())
results = DailyConversionReport.objects.filter(
experiment=experiment).order_by('-date')
mocker.VerifyAll()
self.assertEqual(results.count(), 5)
report_days = [ d.date() for d in days[2:7]]
for i in range(5):
self.assertEqual(results[i].date, report_days[4-i])
# Day 2
self.assertEqual(12, results[4].test_group_size)
self.assertEqual(7, results[4].control_group_size)
self.assertEqual(5, results[4].overall_test_conversion)
self.assertEqual(4, results[4].overall_control_conversion)
day_2_goal_4_test_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[0])[0].test_conversion
self.assertEqual(0, day_2_goal_4_test_conversion)
day_2_goal_2_control_conversion = DailyConversionReportGoalData.objects.filter(
report=results[4],
goal_type=goal_types[2])[0].control_conversion
self.assertEqual(3, day_2_goal_2_control_conversion)
# Day 3
self.assertEqual(5, results[3].test_group_size)
# Day 4
self.assertEqual(0, results[2].test_group_size)
self.assertEqual(None, results[2].confidence)
day_4_goal_1_confidence = DailyConversionReportGoalData.objects.filter(
report=results[2],
goal_type=goal_types[0])[0].confidence
self.assertEqual(None, day_4_goal_1_confidence)
# Day 5
day_5_goal_0_confidence = DailyConversionReportGoalData.objects.filter(
report=results[1],
goal_type=goal_types[0])[0].confidence
self.assertAlmostEqual(98.935467172597029, day_5_goal_0_confidence, places=6)
def testPngResponse(self):
experiment = Experiment(name="test-experiment")
experiment.save()
experiment.state = Experiment.ENABLED_STATE
experiment.save()
goal_type = GoalType(name='test-goal')
goal_type.save()
experiment_url = reverse("django_lean.experiments.tests.views.experiment_test",
args=[experiment.name])
confirm_human_url = reverse("django_lean.experiments.views.confirm_human")
client = Client()
# we can call with invalid or inexisting names, the response is the same
self.call_goal(client, '')
self.call_goal(client, 'unknown-goal')
# this is an anonymous visitor not enrolled in an experiment,
# so no records should be created
self.call_goal(client, goal_type.name)
self.assertEqual(0, GoalRecord.objects.filter(goal_type=goal_type).count())
nb_anonymous_visitors = AnonymousVisitor.objects.count()
# force the user to be a verified human
response = client.get(confirm_human_url)
self.assertEqual(response.status_code, 204)
# force the anonymous visitor to be enrolled in an experiment
response = client.get(experiment_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
client.get('/test-experiment/%s' % experiment.name)
self.assertEqual(nb_anonymous_visitors + 1,
AnonymousVisitor.objects.count())
# now call an existing goal again - it should be recorded
self.call_goal(client, goal_type.name)
self.assertEqual(1, GoalRecord.objects.filter(goal_type=goal_type).count())
# should be recorded again
self.call_goal(client, goal_type.name)
self.assertEqual(2, GoalRecord.objects.filter(goal_type=goal_type).count())
# validate that both of the records have the same anonymous_visitor
two_goal_records = GoalRecord.objects.filter(goal_type=goal_type)
self.assertEqual(two_goal_records[0].anonymous_visitor,
two_goal_records[1].anonymous_visitor)
# try it with a registered user
client = Client()
user = User(username="******", email="*****@*****.**")
user.set_password("password")
user.save()
response = client.login(username=user.username, password='******')
self.assertTrue(response)
# force the registered user to be enrolled in an experiment
client.get('/test-experiment/%s' % experiment.name)
self.call_goal(client, goal_type.name)
# since the user was registered, no new records should be created
self.assertEqual(2, GoalRecord.objects.filter(goal_type=goal_type).count())
