class ConfirmHumanTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
self.experiment_user = participant(session=DatabaseSession())
self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative'])
self.experiment_user.goal('my_goal')
self.redis = get_redis_client()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_confirm_human_updates_experiment(self):
self.assertTrue(self.redis.exists(self.experiment_user._redis_goals_key))
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.assertFalse(self.redis.exists(self.experiment_user._redis_goals_key))
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
def test_confirm_human_called_twice(self):
"""
Ensuring that counters aren't incremented twice
"""
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.experiment_user.confirm_human()
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
def test_confirm_human_sets_session(self):
self.assertFalse(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False))
self.experiment_user.confirm_human()
self.assertTrue(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False))
def test_session_already_confirmed(self):
"""
Testing that confirm_human works even if code outside of django-experiments updates the key
"""
self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True
self.experiment_user.confirm_human()
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
class ConfirmHumanTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
self.experiment_user = participant(session=DatabaseSession())
self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative'])
self.experiment_user.goal('my_goal')
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_confirm_human_updates_experiment(self):
self.assertIn('experiments_goals', self.experiment_user.session)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.assertNotIn('experiments_goals', self.experiment_user.session)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
def test_confirm_human_called_twice(self):
"""
Ensuring that counters aren't incremented twice
"""
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.experiment_user.confirm_human()
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
def test_confirm_human_sets_session(self):
self.assertFalse(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False))
self.experiment_user.confirm_human()
self.assertTrue(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False))
def test_session_already_confirmed(self):
"""
Testing that confirm_human works even if code outside of django-experiments updates the key
"""
self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True
self.experiment_user.confirm_human()
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
class IncorporateTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME,
state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
User = get_user_model()
self.user = User.objects.create(username='******')
self.user.is_confirmed_human = True
request_factory = RequestFactory()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
participant(self.request).confirm_human()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def _login(self):
self.request.user = self.user
transfer_enrollments_to_user(None, self.request, self.user)
def test_visit_incorporate(self):
alternative = participant(self.request).enroll(self.experiment.name,
['alternative'])
ExperimentsRetentionMiddleware().process_response(
self.request, HttpResponse())
self.assertEqual(
dict(
self.experiment_counter.participant_goal_frequencies(
self.experiment, alternative,
participant(self.request)._participant_identifier()))[
conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1)
self.assertFalse(
Enrollment.objects.filter(user__isnull=False).exists())
self._login()
self.assertTrue(Enrollment.objects.filter(user__isnull=False).exists())
self.assertIsNotNone(Enrollment.objects.all()[0].last_seen)
self.assertEqual(
dict(
self.experiment_counter.participant_goal_frequencies(
self.experiment, alternative,
participant(self.request)._participant_identifier()))[
conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1)
self.assertEqual(
self.experiment_counter.goal_count(
self.experiment, alternative,
conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
alternative), 1)
def test_reset_all(self):
experiment = Experiment.objects.create(name='reset_test')
other_experiment = Experiment.objects.create(name='reset_test_other')
experiment_counter = ExperimentCounter()
for exp in [experiment, other_experiment]:
experiment_counter.increment_participant_count(exp, 'alt', 'fred')
experiment_counter.increment_participant_count(exp, 'alt', 'fred')
experiment_counter.increment_participant_count(exp, 'alt', 'fred')
experiment_counter.increment_goal_count(exp, 'alt', 'goal1',
'fred')
experiment_counter.increment_goal_count(exp, 'alt', 'goal2',
'fred')
experiment_counter.increment_participant_count(
exp, 'control', 'barney')
experiment_counter.increment_participant_count(
exp, 'control', 'wilma')
experiment_counter.increment_participant_count(
exp, 'control', 'betty')
experiment_counter.increment_goal_count(exp, 'control', 'goal1',
'betty')
self.counters.reset_prefix(experiment.name)
self.assertEqual(
experiment_counter.participant_count(experiment, 'alt'), 0)
self.assertEqual(
experiment_counter.participant_count(experiment, 'control'), 0)
self.assertEqual(
experiment_counter.goal_count(experiment, 'control', 'goal1'), 0)
self.assertEqual(
experiment_counter.participant_count(other_experiment, 'alt'), 1)
self.assertEqual(
experiment_counter.participant_count(other_experiment, 'control'),
3)
self.assertEqual(
experiment_counter.goal_count(other_experiment, 'control',
'goal1'), 1)
class IncorporateTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
User = get_user_model()
self.user = User.objects.create(username='******')
self.user.is_confirmed_human = True
request_factory = RequestFactory()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
participant(self.request).confirm_human()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def _login(self):
self.request.user = self.user
transfer_enrollments_to_user(None, self.request, self.user)
def test_visit_incorporate(self):
alternative = participant(self.request).enroll(self.experiment.name, ['alternative'])
ExperimentsRetentionMiddleware().process_response(self.request, HttpResponse())
self.assertEqual(
dict(self.experiment_counter.participant_goal_frequencies(self.experiment,
alternative,
participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL],
1
)
self.assertFalse(Enrollment.objects.all().exists())
self._login()
self.assertTrue(Enrollment.objects.all().exists())
self.assertIsNotNone(Enrollment.objects.all()[0].last_seen)
self.assertEqual(
dict(self.experiment_counter.participant_goal_frequencies(self.experiment,
alternative,
participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL],
1
)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, alternative), 1)
def get_result_context(request, experiment):
experiment_counter = ExperimentCounter()
try:
chi2_goals = experiment.relevant_chi2_goals.replace(" ", "").split(",")
except AttributeError:
chi2_goals = ['']
try:
mwu_goals = experiment.relevant_mwu_goals.replace(" ", "").split(",")
except AttributeError:
mwu_goals = ['']
relevant_goals = set(chi2_goals + mwu_goals)
alternatives = {}
for alternative_name in experiment.alternatives.keys():
alternatives[alternative_name] = experiment_counter.participant_count(
experiment, alternative_name)
alternatives = sorted(alternatives.items())
control_participants = experiment_counter.participant_count(
experiment, conf.CONTROL_GROUP)
results = {}
for goal in conf.ALL_GOALS:
show_mwu = goal in mwu_goals
alternatives_conversions = {}
control_conversions = experiment_counter.goal_count(
experiment, conf.CONTROL_GROUP, goal)
control_conversion_rate = rate(control_conversions,
control_participants)
if show_mwu:
mwu_histogram = {}
control_conversion_distribution = fixup_distribution(
experiment_counter.goal_distribution(experiment,
conf.CONTROL_GROUP, goal),
control_participants)
control_average_goal_actions = average_actions(
control_conversion_distribution)
mwu_histogram['control'] = control_conversion_distribution
else:
control_average_goal_actions = None
for alternative_name in experiment.alternatives.keys():
if not alternative_name == conf.CONTROL_GROUP:
alternative_conversions = experiment_counter.goal_count(
experiment, alternative_name, goal)
alternative_participants = experiment_counter.participant_count(
experiment, alternative_name)
alternative_conversion_rate = rate(alternative_conversions,
alternative_participants)
alternative_confidence = chi_squared_confidence(
alternative_participants, alternative_conversions,
control_participants, control_conversions)
if show_mwu:
alternative_conversion_distribution = fixup_distribution(
experiment_counter.goal_distribution(
experiment, alternative_name, goal),
alternative_participants)
alternative_average_goal_actions = average_actions(
alternative_conversion_distribution)
alternative_distribution_confidence = mann_whitney_confidence(
alternative_conversion_distribution,
control_conversion_distribution)
mwu_histogram[
alternative_name] = alternative_conversion_distribution
else:
alternative_average_goal_actions = None
alternative_distribution_confidence = None
alternative = {
'conversions':
alternative_conversions,
'conversion_rate':
alternative_conversion_rate,
'improvement':
improvement(alternative_conversion_rate,
control_conversion_rate),
'confidence':
alternative_confidence,
'average_goal_actions':
alternative_average_goal_actions,
'mann_whitney_confidence':
alternative_distribution_confidence,
}
alternatives_conversions[alternative_name] = alternative
control = {
'conversions': control_conversions,
'conversion_rate': control_conversion_rate,
'average_goal_actions': control_average_goal_actions,
}
results[goal] = {
"control":
control,
"alternatives":
sorted(alternatives_conversions.items()),
"relevant":
goal in relevant_goals or relevant_goals == {''},
"mwu":
goal in mwu_goals,
"mwu_histogram":
conversion_distributions_to_graph_table(mwu_histogram)
if show_mwu else None
}
return {
'experiment': experiment.to_dict(),
'alternatives': alternatives,
'control_participants': control_participants,
'results': results,
'column_count': len(alternatives_conversions) * 3 +
2, # Horrible coupling with template design
'user_alternative':
participant(request).get_alternative(experiment.name),
}
def get_result_context(request, experiment):
experiment_counter = ExperimentCounter()
try:
chi2_goals = experiment.relevant_chi2_goals.replace(" ", "").split(",")
except AttributeError:
chi2_goals = [u'']
try:
mwu_goals = experiment.relevant_mwu_goals.replace(" ", "").split(",")
except AttributeError:
mwu_goals = [u'']
relevant_goals = set(chi2_goals + mwu_goals)
alternatives = {}
for alternative_name in experiment.alternatives.keys():
alternatives[alternative_name] = experiment_counter.participant_count(experiment, alternative_name)
alternatives = sorted(alternatives.items())
control_participants = experiment_counter.participant_count(experiment, conf.CONTROL_GROUP)
results = {}
for goal in conf.ALL_GOALS:
show_mwu = goal in mwu_goals
alternatives_conversions = {}
control_conversions = experiment_counter.goal_count(experiment, conf.CONTROL_GROUP, goal)
control_conversion_rate = rate(control_conversions, control_participants)
if show_mwu:
mwu_histogram = {}
control_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, conf.CONTROL_GROUP, goal), control_participants)
control_average_goal_actions = average_actions(control_conversion_distribution)
mwu_histogram['control'] = control_conversion_distribution
else:
control_average_goal_actions = None
for alternative_name in experiment.alternatives.keys():
if not alternative_name == conf.CONTROL_GROUP:
alternative_conversions = experiment_counter.goal_count(experiment, alternative_name, goal)
alternative_participants = experiment_counter.participant_count(experiment, alternative_name)
alternative_conversion_rate = rate(alternative_conversions, alternative_participants)
alternative_confidence = chi_squared_confidence(alternative_participants, alternative_conversions, control_participants, control_conversions)
if show_mwu:
alternative_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, alternative_name, goal), alternative_participants)
alternative_average_goal_actions = average_actions(alternative_conversion_distribution)
alternative_distribution_confidence = mann_whitney_confidence(alternative_conversion_distribution, control_conversion_distribution)
mwu_histogram[alternative_name] = alternative_conversion_distribution
else:
alternative_average_goal_actions = None
alternative_distribution_confidence = None
alternative = {
'conversions': alternative_conversions,
'conversion_rate': alternative_conversion_rate,
'improvement': improvement(alternative_conversion_rate, control_conversion_rate),
'confidence': alternative_confidence,
'average_goal_actions': alternative_average_goal_actions,
'mann_whitney_confidence': alternative_distribution_confidence,
}
alternatives_conversions[alternative_name] = alternative
control = {
'conversions': control_conversions,
'conversion_rate': control_conversion_rate,
'average_goal_actions': control_average_goal_actions,
}
results[goal] = {
"control": control,
"alternatives": sorted(alternatives_conversions.items()),
"relevant": goal in relevant_goals or relevant_goals == {u''},
"mwu": goal in mwu_goals,
"mwu_histogram": conversion_distributions_to_graph_table(mwu_histogram) if show_mwu else None
}
return {
'experiment': experiment.to_dict(),
'alternatives': alternatives,
'control_participants': control_participants,
'results': results,
'column_count': len(alternatives_conversions) * 3 + 2, # Horrible coupling with template design
'user_alternative': participant(request).get_alternative(experiment.name),
}
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_enrollment_initially_control(self):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control")
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set")
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count")
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution")
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_visit_increases_goal(self):
thetime = timezone.now()
with patch('experiments.utils.now', return_value=thetime):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted")
with patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)):
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted")
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted")
def test_user_force_enrolls(self):
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2')
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2')
def test_user_does_not_force_enroll_to_new_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives)
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative)
self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
def test_second_force_enroll_does_not_change_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1')
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative)
self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get("/")
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment_counter.participant_count(self.experiment, alternative)
def enrollment_initially_none(self,):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), None)
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_visit_increases_goal(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}
)
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}
)
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_enrollment_initially_control(self):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
'control', "Default Enrollment wasn't control")
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
TEST_ALTERNATIVE, "Wrong Alternative Set")
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1,
"Did not increment Goal count")
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1,
"Did not increment goal count correctly")
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment,
TEST_ALTERNATIVE,
TEST_GOAL), {3: 1},
"Incorrect goal count distribution")
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
TEST_ALTERNATIVE), 1,
"Did not count participant after confirm human")
def test_visit_increases_goal(self):
thetime = timezone.now()
with mock.patch('experiments.utils.now', return_value=thetime):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1},
"Not Present Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_PRESENT_COUNT_GOAL), {},
"Present Visit was not correctly counted")
with mock.patch('experiments.utils.now',
return_value=thetime + timedelta(hours=7)):
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1},
"No Present Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_PRESENT_COUNT_GOAL), {1: 1},
"Present Visit was not correctly counted")
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1},
"Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL),
{}, "Present Visit was not correctly counted")
def test_user_force_enrolls(self):
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME,
['control', 'alternative1', 'alternative2'],
force_alternative='alternative2')
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
'alternative2')
def test_user_does_not_force_enroll_to_new_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives)
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(
list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative=other_alternative)
self.assertEqual(alternative,
experiment_user.get_alternative(EXPERIMENT_NAME))
def test_second_force_enroll_does_not_change_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative='alternative1')
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(
list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative=other_alternative)
self.assertEqual(alternative,
experiment_user.get_alternative(EXPERIMENT_NAME))
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment_counter.participant_count(
self.experiment, alternative)
def enrollment_initially_none(self, ):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
None)
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1,
"Did not count participant after confirm human")
def test_visit_increases_goal(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment,
TEST_ALTERNATIVE,
VISIT_COUNT_GOAL),
{1: 1})
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment,
TEST_ALTERNATIVE,
VISIT_COUNT_GOAL),
{1: 1})