class BotTestCase(TestCase):
def setUp(self):
self.experiment = Experiment(name="backgroundcolor", state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get("/", HTTP_USER_AGENT="GoogleBot/2.1")
self.experiment_counter = ExperimentCounter()
def test_user_does_not_enroll(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE),
0,
"Bot counted towards results",
)
def test_bot_in_control_group(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), "control", "Bot enrolled in a group")
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE, self.request),
False,
"Bot in test alternative",
)
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP, self.request),
True,
"Bot not in control group",
)
def tearDown(self):
self.experiment_counter.delete(self.experiment)
class WebUserIncorporateTestCase(object):
def __init__(self, *args, **kwargs):
super(WebUserIncorporateTestCase, self).__init__(*args, **kwargs)
self.experiment_counter = ExperimentCounter()
def test_can_incorporate(self):
self.incorporating.incorporate(self.incorporated)
def test_incorporates_enrollment_from_other(self):
if not self._has_data():
return
try:
experiment = Experiment.objects.create(name=EXPERIMENT_NAME,
state=ENABLED_STATE)
self.incorporated.set_alternative(EXPERIMENT_NAME,
TEST_ALTERNATIVE)
self.incorporating.incorporate(self.incorporated)
self.assertEqual(
self.incorporating.get_alternative(EXPERIMENT_NAME),
TEST_ALTERNATIVE)
finally:
self.experiment_counter.delete(experiment)
def _has_data(self):
return not isinstance(self.incorporated, DummyUser) and not isinstance(
self.incorporating, DummyUser)
def setUp(self):
self.experiment = Experiment(name='backgroundcolor',
state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/',
HTTP_USER_AGENT='GoogleBot/2.1')
self.experiment_counter = ExperimentCounter()
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
self.experiment_user = participant(session=DatabaseSession())
self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative'])
self.experiment_user.goal('my_goal')
self.redis = get_redis_client()
class BotTestCase(TestCase):
def setUp(self):
self.experiment = Experiment(name='backgroundcolor',
state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/',
HTTP_USER_AGENT='GoogleBot/2.1')
self.experiment_counter = ExperimentCounter()
def test_user_does_not_enroll(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
TEST_ALTERNATIVE), 0,
"Bot counted towards results")
def test_bot_in_control_group(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
'control', "Bot enrolled in a group")
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE,
self.request), False,
"Bot in test alternative")
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP,
self.request), True,
"Bot not in control group")
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def delete(self, request):
if not request.user.has_perm('experiments.delete_experiment'):
raise ExperimentException("You don't have permission to do that!")
experiment_counter = ExperimentCounter()
experiment = Experiment.objects.get(name=request.POST.get("name"))
experiment.enrollment_set.all().delete()
experiment_counter.delete(experiment)
experiment.delete()
return {'successful': True}
def delete(self, request):
if not request.user.has_perm('experiments.delete_experiment'):
raise ExperimentException("You don't have permission to do that!")
experiment_counter = ExperimentCounter()
experiment = Experiment.objects.get(name=request.POST.get("name"))
experiment.enrollment_set.all().delete()
experiment_counter.delete(experiment)
experiment.delete()
return {'successful': True}
def setUp(self):
self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME,
state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
User = get_user_model()
self.user = User.objects.create(username='******')
self.user.is_confirmed_human = True
request_factory = RequestFactory()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
participant(self.request).confirm_human()
def setUp(self):
self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
User = get_user_model()
self.user = User.objects.create(username='******')
self.user.is_confirmed_human = True
request_factory = RequestFactory()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
participant(self.request).confirm_human()
class ParticipantCacheTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_transfer_enrollments(self):
User = get_user_model()
user = User.objects.create(username='******')
request = request_factory.get('/')
request.session = DatabaseSession()
participant(request).enroll('test_experiment1', ['alternative'])
request.user = user
transfer_enrollments_to_user(None, request, user)
# the call to the middleware will set last_seen on the experiment
# if the participant cache hasn't been wiped appropriately then the
# session experiment user will be impacted instead of the authenticated
# experiment user
ExperimentsRetentionMiddleware().process_response(request, HttpResponse())
self.assertIsNotNone(Enrollment.objects.all()[0].last_seen)
class ParticipantCacheTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_transfer_enrollments(self):
User = get_user_model()
user = User.objects.create(username='******')
request = request_factory.get('/')
request.session = DatabaseSession()
participant(request).enroll('test_experiment1', ['alternative'])
request.user = user
transfer_enrollments_to_user(None, request, user)
# the call to the middleware will set last_seen on the experiment
# if the participant cache hasn't been wiped appropriately then the
# session experiment user will be impacted instead of the authenticated
# experiment user
ExperimentsRetentionMiddleware().process_response(request, HttpResponse())
self.assertIsNotNone(Enrollment.objects.all()[0].last_seen)
class WebUserIncorporateTestCase(object):
def __init__(self, *args, **kwargs):
super(WebUserIncorporateTestCase, self).__init__(*args, **kwargs)
self.experiment_counter = ExperimentCounter()
def test_can_incorporate(self):
self.incorporating.incorporate(self.incorporated)
def test_incorporates_enrollment_from_other(self):
if not self._has_data():
return
try:
experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.incorporated.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.incorporating.incorporate(self.incorporated)
self.assertEqual(self.incorporating.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE)
finally:
self.experiment_counter.delete(experiment)
def _has_data(self):
return not isinstance(self.incorporated, DummyUser) and not isinstance(self.incorporating, DummyUser)
class BotTests(object):
def setUp(self):
self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE)
self.experiment.save()
self.experiment_counter = ExperimentCounter()
def test_user_does_not_enroll(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_user_does_not_fire_goals(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_bot_in_control_group(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Bot enrolled in a group")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE), False, "Bot in test alternative")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP), True, "Bot not in control group")
def tearDown(self):
self.experiment_counter.delete(self.experiment)
class BotTests(object):
def setUp(self):
self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE)
self.experiment.save()
self.experiment_counter = ExperimentCounter()
def test_user_does_not_enroll(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_user_does_not_fire_goals(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_bot_in_control_group(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Bot enrolled in a group")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE), False, "Bot in test alternative")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP), True, "Bot not in control group")
def tearDown(self):
self.experiment_counter.delete(self.experiment)
class IncorporateTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME,
state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
User = get_user_model()
self.user = User.objects.create(username='******')
self.user.is_confirmed_human = True
request_factory = RequestFactory()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
participant(self.request).confirm_human()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def _login(self):
self.request.user = self.user
transfer_enrollments_to_user(None, self.request, self.user)
def test_visit_incorporate(self):
alternative = participant(self.request).enroll(self.experiment.name,
['alternative'])
ExperimentsRetentionMiddleware().process_response(
self.request, HttpResponse())
self.assertEqual(
dict(
self.experiment_counter.participant_goal_frequencies(
self.experiment, alternative,
participant(self.request)._participant_identifier()))[
conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1)
self.assertFalse(
Enrollment.objects.filter(user__isnull=False).exists())
self._login()
self.assertTrue(Enrollment.objects.filter(user__isnull=False).exists())
self.assertIsNotNone(Enrollment.objects.all()[0].last_seen)
self.assertEqual(
dict(
self.experiment_counter.participant_goal_frequencies(
self.experiment, alternative,
participant(self.request)._participant_identifier()))[
conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1)
self.assertEqual(
self.experiment_counter.goal_count(
self.experiment, alternative,
conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
alternative), 1)
class IncorporateTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
User = get_user_model()
self.user = User.objects.create(username='******')
self.user.is_confirmed_human = True
request_factory = RequestFactory()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
participant(self.request).confirm_human()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def _login(self):
self.request.user = self.user
transfer_enrollments_to_user(None, self.request, self.user)
def test_visit_incorporate(self):
alternative = participant(self.request).enroll(self.experiment.name, ['alternative'])
ExperimentsRetentionMiddleware().process_response(self.request, HttpResponse())
self.assertEqual(
dict(self.experiment_counter.participant_goal_frequencies(self.experiment,
alternative,
participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL],
1
)
self.assertFalse(Enrollment.objects.all().exists())
self._login()
self.assertTrue(Enrollment.objects.all().exists())
self.assertIsNotNone(Enrollment.objects.all()[0].last_seen)
self.assertEqual(
dict(self.experiment_counter.participant_goal_frequencies(self.experiment,
alternative,
participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL],
1
)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, alternative), 1)
def __init__(self, *args, **kwargs):
super(WebUserIncorporateTestCase, self).__init__(*args, **kwargs)
self.experiment_counter = ExperimentCounter()
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_enrollment_initially_control(self):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
'control', "Default Enrollment wasn't control")
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
TEST_ALTERNATIVE, "Wrong Alternative Set")
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1,
"Did not increment Goal count")
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1,
"Did not increment goal count correctly")
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment,
TEST_ALTERNATIVE,
TEST_GOAL), {3: 1},
"Incorrect goal count distribution")
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
TEST_ALTERNATIVE), 1,
"Did not count participant after confirm human")
def test_visit_increases_goal(self):
thetime = timezone.now()
with mock.patch('experiments.utils.now', return_value=thetime):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1},
"Not Present Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_PRESENT_COUNT_GOAL), {},
"Present Visit was not correctly counted")
with mock.patch('experiments.utils.now',
return_value=thetime + timedelta(hours=7)):
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1},
"No Present Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_PRESENT_COUNT_GOAL), {1: 1},
"Present Visit was not correctly counted")
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1},
"Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL),
{}, "Present Visit was not correctly counted")
def test_user_force_enrolls(self):
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME,
['control', 'alternative1', 'alternative2'],
force_alternative='alternative2')
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
'alternative2')
def test_user_does_not_force_enroll_to_new_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives)
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(
list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative=other_alternative)
self.assertEqual(alternative,
experiment_user.get_alternative(EXPERIMENT_NAME))
def test_second_force_enroll_does_not_change_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative='alternative1')
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(
list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative=other_alternative)
self.assertEqual(alternative,
experiment_user.get_alternative(EXPERIMENT_NAME))
class ConfirmHumanTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1',
state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
self.experiment_user = participant(session=DatabaseSession())
self.alternative = self.experiment_user.enroll(self.experiment.name,
['alternative'])
self.experiment_user.goal('my_goal')
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_confirm_human_updates_experiment(self):
self.assertIn('experiments_goals', self.experiment_user.session)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
self.alternative), 0)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.assertNotIn('experiments_goals', self.experiment_user.session)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
self.alternative), 1)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
self.alternative, 'my_goal'), 1)
def test_confirm_human_called_twice(self):
"""
Ensuring that counters aren't incremented twice
"""
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
self.alternative), 0)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.experiment_user.confirm_human()
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
self.alternative), 1)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
self.alternative, 'my_goal'), 1)
def test_confirm_human_sets_session(self):
self.assertFalse(
self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY,
False))
self.experiment_user.confirm_human()
self.assertTrue(
self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY,
False))
def test_session_already_confirmed(self):
"""
Testing that confirm_human works even if code outside of django-experiments updates the key
"""
self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True
self.experiment_user.confirm_human()
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
self.alternative), 1)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
self.alternative, 'my_goal'), 1)
def __init__(self):
self.experiment_counter = ExperimentCounter()
self.experiments_exposure = []
class WebUser(object):
"""Represents a user (either authenticated or session based) which can take part in experiments"""
def __init__(self):
self.experiment_counter = ExperimentCounter()
def enroll(self, experiment_name, alternatives, force_alternative=None):
"""
Enroll this user in the experiment if they are not already part of it. Returns the selected alternative
force_alternative: Optionally force a user in an alternative at enrollment time
"""
chosen_alternative = conf.CONTROL_GROUP
experiment = experiment_manager.get_experiment(experiment_name)
if experiment:
if experiment.is_displaying_alternatives():
if isinstance(alternatives, collections.Mapping):
if conf.CONTROL_GROUP not in alternatives:
experiment.ensure_alternative_exists(
conf.CONTROL_GROUP, 1)
for alternative, weight in alternatives.items():
experiment.ensure_alternative_exists(
alternative, weight)
else:
alternatives_including_control = alternatives + [
conf.CONTROL_GROUP
]
for alternative in alternatives_including_control:
experiment.ensure_alternative_exists(alternative)
assigned_alternative = self._get_enrollment(experiment)
if assigned_alternative:
chosen_alternative = assigned_alternative
elif experiment.is_accepting_new_users():
if force_alternative:
chosen_alternative = force_alternative
else:
chosen_alternative = experiment.random_alternative()
self._set_enrollment(experiment, chosen_alternative)
else:
chosen_alternative = experiment.default_alternative
return chosen_alternative
def get_alternative(self, experiment_name):
"""
Get the alternative this user is enrolled in.
"""
experiment = None
try:
# catching the KeyError instead of using .get so that the experiment is auto created if desired
experiment = experiment_manager[experiment_name]
except KeyError:
pass
if experiment:
if experiment.is_displaying_alternatives():
alternative = self._get_enrollment(experiment)
if alternative is not None:
return alternative
else:
return experiment.default_alternative
return conf.CONTROL_GROUP
def set_alternative(self, experiment_name, alternative):
"""Explicitly set the alternative the user is enrolled in for the specified experiment.
This allows you to change a user between alternatives. The user and goal counts for the new
alternative will be increment, but those for the old one will not be decremented. The user will
be enrolled in the experiment even if the experiment would not normally accept this user."""
experiment = experiment_manager.get_experiment(experiment_name)
if experiment:
self._set_enrollment(experiment, alternative)
def goal(self, goal_name, count=1):
"""Record that this user has performed a particular goal
This will update the goal stats for all experiments the user is enrolled in."""
for enrollment in self._get_all_enrollments():
if enrollment.experiment.is_displaying_alternatives():
self._experiment_goal(enrollment.experiment,
enrollment.alternative, goal_name, count)
def confirm_human(self):
"""Mark that this is a real human being (not a bot) and thus results should be counted"""
pass
def incorporate(self, other_user):
"""Incorporate all enrollments and goals performed by the other user
If this user is not enrolled in a given experiment, the results for the
other user are incorporated. For experiments this user is already
enrolled in the results of the other user are discarded.
This takes a relatively large amount of time for each experiment the other
user is enrolled in."""
for enrollment in other_user._get_all_enrollments():
if not self._get_enrollment(enrollment.experiment):
self._set_enrollment(enrollment.experiment,
enrollment.alternative,
enrollment.enrollment_date,
enrollment.last_seen)
goals = self.experiment_counter.participant_goal_frequencies(
enrollment.experiment, enrollment.alternative,
other_user._participant_identifier())
for goal_name, count in goals:
self.experiment_counter.increment_goal_count(
enrollment.experiment, enrollment.alternative,
goal_name, self._participant_identifier(), count)
other_user._cancel_enrollment(enrollment.experiment)
def visit(self):
"""Record that the user has visited the site for the purposes of retention tracking"""
for enrollment in self._get_all_enrollments():
if enrollment.experiment.is_displaying_alternatives():
# We have two different goals, VISIT_NOT_PRESENT_COUNT_GOAL and VISIT_PRESENT_COUNT_GOAL.
# VISIT_PRESENT_COUNT_GOAL will avoid firing on the first time we set last_seen as it is assumed that the user is
# on the page and therefore it would automatically trigger and be valueless.
# This should be used for experiments when we enroll the user as part of the pageview,
# alternatively we can use the NOT_PRESENT GOAL which will increment on the first pageview,
# this is mainly useful for notification actions when the users isn't initially present.
if not enrollment.last_seen:
self._experiment_goal(enrollment.experiment,
enrollment.alternative,
conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1)
self._set_last_seen(enrollment.experiment, now())
elif now() - enrollment.last_seen >= timedelta(
hours=conf.SESSION_LENGTH):
self._experiment_goal(enrollment.experiment,
enrollment.alternative,
conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1)
self._experiment_goal(enrollment.experiment,
enrollment.alternative,
conf.VISIT_PRESENT_COUNT_GOAL, 1)
self._set_last_seen(enrollment.experiment, now())
def _get_enrollment(self, experiment):
"""Get the name of the alternative this user is enrolled in for the specified experiment
`experiment` is an instance of Experiment. If the user is not currently enrolled returns None."""
raise NotImplementedError
def _set_enrollment(self,
experiment,
alternative,
enrollment_date=None,
last_seen=None):
"""Explicitly set the alternative the user is enrolled in for the specified experiment.
This allows you to change a user between alternatives. The user and goal counts for the new
alternative will be increment, but those for the old one will not be decremented."""
raise NotImplementedError
def is_enrolled(self, experiment_name, alternative):
"""Enroll this user in the experiment if they are not already part of it. Returns the selected alternative"""
"""Test if the user is enrolled in the supplied alternative for the given experiment.
The supplied alternative will be added to the list of possible alternatives for the
experiment if it is not already there. If the user is not yet enrolled in the supplied
experiment they will be enrolled, and an alternative chosen at random."""
chosen_alternative = self.enroll(experiment_name, [alternative])
return alternative == chosen_alternative
def _participant_identifier(self):
"Unique identifier for this user in the counter store"
raise NotImplementedError
def _get_all_enrollments(self):
"Return experiment, alternative tuples for all experiments the user is enrolled in"
raise NotImplementedError
def _cancel_enrollment(self, experiment):
"Remove the enrollment and any goals the user has against this experiment"
raise NotImplementedError
def _experiment_goal(self, experiment, alternative, goal_name, count):
"Record a goal against a particular experiment and alternative"
raise NotImplementedError
def _set_last_seen(self, experiment, last_seen):
"Set the last time the user was seen associated with this experiment"
raise NotImplementedError
class WebUser(object):
"""Represents a user (either authenticated or session based) which can take part in experiments"""
def __init__(self):
self.experiment_counter = ExperimentCounter()
def enroll(self, experiment_name, alternatives, force_alternative=None):
"""
Enroll this user in the experiment if they are not already part of it. Returns the selected alternative
force_alternative: Optionally force a user in an alternative at enrollment time
"""
chosen_alternative = conf.CONTROL_GROUP
experiment = experiment_manager.get(experiment_name, None)
if experiment and experiment.is_displaying_alternatives():
if isinstance(alternatives, collections.Mapping):
if conf.CONTROL_GROUP not in alternatives:
experiment.ensure_alternative_exists(conf.CONTROL_GROUP, 1)
for alternative, weight in alternatives.items():
experiment.ensure_alternative_exists(alternative, weight)
else:
alternatives_including_control = alternatives + [conf.CONTROL_GROUP]
for alternative in alternatives_including_control:
experiment.ensure_alternative_exists(alternative)
assigned_alternative = self._get_enrollment(experiment)
if assigned_alternative:
chosen_alternative = assigned_alternative
elif experiment.is_accepting_new_users():
if force_alternative:
chosen_alternative = force_alternative
else:
chosen_alternative = experiment.random_alternative()
self._set_enrollment(experiment, chosen_alternative)
return chosen_alternative
def get_alternative(self, experiment_name):
"""Get the alternative this user is enrolled in. If not enrolled in the experiment returns 'control'"""
experiment = experiment_manager.get(experiment_name, None)
if experiment and experiment.is_displaying_alternatives():
alternative = self._get_enrollment(experiment)
if alternative is not None:
return alternative
return "control"
def set_alternative(self, experiment_name, alternative):
"""Explicitly set the alternative the user is enrolled in for the specified experiment.
This allows you to change a user between alternatives. The user and goal counts for the new
alternative will be increment, but those for the old one will not be decremented. The user will
be enrolled in the experiment even if the experiment would not normally accept this user."""
experiment = experiment_manager.get(experiment_name, None)
if experiment:
self._set_enrollment(experiment, alternative)
def goal(self, goal_name, count=1):
"""Record that this user has performed a particular goal
This will update the goal stats for all experiments the user is enrolled in."""
for enrollment in self._get_all_enrollments():
if enrollment.experiment.is_displaying_alternatives():
self._experiment_goal(enrollment.experiment, enrollment.alternative, goal_name, count)
def confirm_human(self):
"""Mark that this is a real human being (not a bot) and thus results should be counted"""
pass
def incorporate(self, other_user):
"""Incorporate all enrollments and goals performed by the other user
If this user is not enrolled in a given experiment, the results for the
other user are incorporated. For experiments this user is already
enrolled in the results of the other user are discarded.
This takes a relatively large amount of time for each experiment the other
user is enrolled in."""
for enrollment in other_user._get_all_enrollments():
if not self._get_enrollment(enrollment.experiment):
self._set_enrollment(
enrollment.experiment, enrollment.alternative, enrollment.enrollment_date, enrollment.last_seen
)
goals = self.experiment_counter.participant_goal_frequencies(
enrollment.experiment, enrollment.alternative, other_user._participant_identifier()
)
for goal_name, count in goals:
self.experiment_counter.increment_goal_count(
enrollment.experiment, enrollment.alternative, goal_name, self._participant_identifier(), count
)
other_user._cancel_enrollment(enrollment.experiment)
def visit(self):
"""Record that the user has visited the site for the purposes of retention tracking"""
for enrollment in self._get_all_enrollments():
if enrollment.experiment.is_displaying_alternatives():
# We have two different goals, VISIT_NOT_PRESENT_COUNT_GOAL and VISIT_PRESENT_COUNT_GOAL.
# VISIT_PRESENT_COUNT_GOAL will avoid firing on the first time we set last_seen as it is assumed that the user is
# on the page and therefore it would automatically trigger and be valueless.
# This should be used for experiments when we enroll the user as part of the pageview,
# alternatively we can use the NOT_PRESENT GOAL which will increment on the first pageview,
# this is mainly useful for notification actions when the users isn't initially present.
if not enrollment.last_seen:
self._experiment_goal(
enrollment.experiment, enrollment.alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1
)
self._set_last_seen(enrollment.experiment, now())
elif now() - enrollment.last_seen >= timedelta(hours=conf.SESSION_LENGTH):
self._experiment_goal(
enrollment.experiment, enrollment.alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1
)
self._experiment_goal(
enrollment.experiment, enrollment.alternative, conf.VISIT_PRESENT_COUNT_GOAL, 1
)
self._set_last_seen(enrollment.experiment, now())
def _get_enrollment(self, experiment):
"""Get the name of the alternative this user is enrolled in for the specified experiment
`experiment` is an instance of Experiment. If the user is not currently enrolled returns None."""
raise NotImplementedError
def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None):
"""Explicitly set the alternative the user is enrolled in for the specified experiment.
This allows you to change a user between alternatives. The user and goal counts for the new
alternative will be increment, but those for the old one will not be decremented."""
raise NotImplementedError
def is_enrolled(self, experiment_name, alternative):
"""Enroll this user in the experiment if they are not already part of it. Returns the selected alternative"""
"""Test if the user is enrolled in the supplied alternative for the given experiment.
The supplied alternative will be added to the list of possible alternatives for the
experiment if it is not already there. If the user is not yet enrolled in the supplied
experiment they will be enrolled, and an alternative chosen at random."""
chosen_alternative = self.enroll(experiment_name, [alternative])
return alternative == chosen_alternative
def _participant_identifier(self):
"Unique identifier for this user in the counter store"
raise NotImplementedError
def _get_all_enrollments(self):
"Return experiment, alternative tuples for all experiments the user is enrolled in"
raise NotImplementedError
def _cancel_enrollment(self, experiment):
"Remove the enrollment and any goals the user has against this experiment"
raise NotImplementedError
def _experiment_goal(self, experiment, alternative, goal_name, count):
"Record a goal against a particular experiment and alternative"
raise NotImplementedError
def _set_last_seen(self, experiment, last_seen):
"Set the last time the user was seen associated with this experiment"
raise NotImplementedError
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment_counter.participant_count(
self.experiment, alternative)
def enrollment_initially_none(self, ):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
None)
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1,
"Did not count participant after confirm human")
def test_visit_increases_goal(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment,
TEST_ALTERNATIVE,
VISIT_COUNT_GOAL),
{1: 1})
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment,
TEST_ALTERNATIVE,
VISIT_COUNT_GOAL),
{1: 1})
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1',
state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
def test_reset_all(self):
experiment = Experiment.objects.create(name='reset_test')
other_experiment = Experiment.objects.create(name='reset_test_other')
experiment_counter = ExperimentCounter()
for exp in [experiment, other_experiment]:
experiment_counter.increment_participant_count(exp, 'alt', 'fred')
experiment_counter.increment_participant_count(exp, 'alt', 'fred')
experiment_counter.increment_participant_count(exp, 'alt', 'fred')
experiment_counter.increment_goal_count(exp, 'alt', 'goal1',
'fred')
experiment_counter.increment_goal_count(exp, 'alt', 'goal2',
'fred')
experiment_counter.increment_participant_count(
exp, 'control', 'barney')
experiment_counter.increment_participant_count(
exp, 'control', 'wilma')
experiment_counter.increment_participant_count(
exp, 'control', 'betty')
experiment_counter.increment_goal_count(exp, 'control', 'goal1',
'betty')
self.counters.reset_prefix(experiment.name)
self.assertEqual(
experiment_counter.participant_count(experiment, 'alt'), 0)
self.assertEqual(
experiment_counter.participant_count(experiment, 'control'), 0)
self.assertEqual(
experiment_counter.goal_count(experiment, 'control', 'goal1'), 0)
self.assertEqual(
experiment_counter.participant_count(other_experiment, 'alt'), 1)
self.assertEqual(
experiment_counter.participant_count(other_experiment, 'control'),
3)
self.assertEqual(
experiment_counter.goal_count(other_experiment, 'control',
'goal1'), 1)
def setUp(self):
self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE)
self.experiment.save()
self.experiment_counter = ExperimentCounter()
def setUp(self):
self.experiment = Experiment(name="backgroundcolor", state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get("/", HTTP_USER_AGENT="GoogleBot/2.1")
self.experiment_counter = ExperimentCounter()
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get("/")
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment_counter.participant_count(self.experiment, alternative)
def enrollment_initially_none(self,):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), None)
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_visit_increases_goal(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}
)
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}
)
def setUp(self):
self.experiment = Experiment(name='backgroundcolor',
state=ENABLED_STATE)
self.experiment.save()
self.experiment_counter = ExperimentCounter()
class ConfirmHumanTestCase(TestCase):
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
self.experiment_user = participant(session=DatabaseSession())
self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative'])
self.experiment_user.goal('my_goal')
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_confirm_human_updates_experiment(self):
self.assertIn('experiments_goals', self.experiment_user.session)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.assertNotIn('experiments_goals', self.experiment_user.session)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
def test_confirm_human_called_twice(self):
"""
Ensuring that counters aren't incremented twice
"""
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0)
self.experiment_user.confirm_human()
self.experiment_user.confirm_human()
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
def test_confirm_human_sets_session(self):
self.assertFalse(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False))
self.experiment_user.confirm_human()
self.assertTrue(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False))
def test_session_already_confirmed(self):
"""
Testing that confirm_human works even if code outside of django-experiments updates the key
"""
self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True
self.experiment_user.confirm_human()
self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
def __init__(self, *args, **kwargs):
super(WebUserIncorporateTestCase, self).__init__(*args, **kwargs)
self.experiment_counter = ExperimentCounter()
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
self.experiment_user = participant(session=DatabaseSession())
self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative'])
self.experiment_user.goal('my_goal')
def get_result_context(request, experiment):
experiment_counter = ExperimentCounter()
try:
chi2_goals = experiment.relevant_chi2_goals.replace(" ", "").split(",")
except AttributeError:
chi2_goals = [u'']
try:
mwu_goals = experiment.relevant_mwu_goals.replace(" ", "").split(",")
except AttributeError:
mwu_goals = [u'']
relevant_goals = set(chi2_goals + mwu_goals)
alternatives = {}
for alternative_name in experiment.alternatives.keys():
alternatives[alternative_name] = experiment_counter.participant_count(experiment, alternative_name)
alternatives = sorted(alternatives.items())
control_participants = experiment_counter.participant_count(experiment, conf.CONTROL_GROUP)
results = {}
for goal in conf.ALL_GOALS:
show_mwu = goal in mwu_goals
alternatives_conversions = {}
control_conversions = experiment_counter.goal_count(experiment, conf.CONTROL_GROUP, goal)
control_conversion_rate = rate(control_conversions, control_participants)
if show_mwu:
mwu_histogram = {}
control_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, conf.CONTROL_GROUP, goal), control_participants)
control_average_goal_actions = average_actions(control_conversion_distribution)
mwu_histogram['control'] = control_conversion_distribution
else:
control_average_goal_actions = None
for alternative_name in experiment.alternatives.keys():
if not alternative_name == conf.CONTROL_GROUP:
alternative_conversions = experiment_counter.goal_count(experiment, alternative_name, goal)
alternative_participants = experiment_counter.participant_count(experiment, alternative_name)
alternative_conversion_rate = rate(alternative_conversions, alternative_participants)
alternative_confidence = chi_squared_confidence(alternative_participants, alternative_conversions, control_participants, control_conversions)
if show_mwu:
alternative_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, alternative_name, goal), alternative_participants)
alternative_average_goal_actions = average_actions(alternative_conversion_distribution)
alternative_distribution_confidence = mann_whitney_confidence(alternative_conversion_distribution, control_conversion_distribution)
mwu_histogram[alternative_name] = alternative_conversion_distribution
else:
alternative_average_goal_actions = None
alternative_distribution_confidence = None
alternative = {
'conversions': alternative_conversions,
'conversion_rate': alternative_conversion_rate,
'improvement': improvement(alternative_conversion_rate, control_conversion_rate),
'confidence': alternative_confidence,
'average_goal_actions': alternative_average_goal_actions,
'mann_whitney_confidence': alternative_distribution_confidence,
}
alternatives_conversions[alternative_name] = alternative
control = {
'conversions': control_conversions,
'conversion_rate': control_conversion_rate,
'average_goal_actions': control_average_goal_actions,
}
results[goal] = {
"control": control,
"alternatives": sorted(alternatives_conversions.items()),
"relevant": goal in relevant_goals or relevant_goals == {u''},
"mwu": goal in mwu_goals,
"mwu_histogram": conversion_distributions_to_graph_table(mwu_histogram) if show_mwu else None
}
return {
'experiment': experiment.to_dict(),
'alternatives': alternatives,
'control_participants': control_participants,
'results': results,
'column_count': len(alternatives_conversions) * 3 + 2, # Horrible coupling with template design
'user_alternative': participant(request).get_alternative(experiment.name),
}
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_enrollment_initially_control(self):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control")
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set")
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count")
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution")
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_visit_increases_goal(self):
thetime = timezone.now()
with patch('experiments.utils.now', return_value=thetime):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted")
with patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)):
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted")
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted")
def test_user_force_enrolls(self):
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2')
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2')
def test_user_does_not_force_enroll_to_new_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives)
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative)
self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
def test_second_force_enroll_does_not_change_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1')
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative)
self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
def __init__(self):
self.experiment_counter = ExperimentCounter()
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def __init__(self):
self.experiment_counter = ExperimentCounter()
class WebUser(object):
"""Represents a user (either authenticated or session based) which can take part in experiments"""
def __init__(self):
self.experiment_counter = ExperimentCounter()
def enroll(self, experiment_name, alternatives):
"""Enroll this user in the experiment if they are not already part of it. Returns the selected alternative"""
chosen_alternative = conf.CONTROL_GROUP
experiment = experiment_manager.get(experiment_name, None)
if experiment and experiment.is_displaying_alternatives():
if isinstance(alternatives, collections.Mapping):
if conf.CONTROL_GROUP not in alternatives:
experiment.ensure_alternative_exists(conf.CONTROL_GROUP, 1)
for alternative, weight in alternatives.items():
experiment.ensure_alternative_exists(alternative, weight)
else:
alternatives_including_control = alternatives + [conf.CONTROL_GROUP]
for alternative in alternatives_including_control:
experiment.ensure_alternative_exists(alternative)
assigned_alternative = self._get_enrollment(experiment)
if assigned_alternative:
chosen_alternative = assigned_alternative
elif experiment.is_accepting_new_users(self._gargoyle_key()):
chosen_alternative = experiment.random_alternative()
self._set_enrollment(experiment, chosen_alternative)
return chosen_alternative
def get_alternative(self, experiment_name):
"""Get the alternative this user is enrolled in. If not enrolled in the experiment returns 'control'"""
experiment = experiment_manager.get(experiment_name, None)
if experiment and experiment.is_displaying_alternatives():
alternative = self._get_enrollment(experiment)
if alternative is not None:
return alternative
return 'control'
def set_alternative(self, experiment_name, alternative):
"""Explicitly set the alternative the user is enrolled in for the specified experiment.
This allows you to change a user between alternatives. The user and goal counts for the new
alternative will be increment, but those for the old one will not be decremented. The user will
be enrolled in the experiment even if the experiment would not normally accept this user."""
experiment = experiment_manager.get(experiment_name, None)
if experiment:
self._set_enrollment(experiment, alternative)
def goal(self, goal_name, count=1):
"""Record that this user has performed a particular goal
This will update the goal stats for all experiments the user is enrolled in."""
for enrollment in self._get_all_enrollments():
if enrollment.experiment.is_displaying_alternatives():
self._experiment_goal(enrollment.experiment, enrollment.alternative, goal_name, count)
def confirm_human(self):
"""Mark that this is a real human being (not a bot) and thus results should be counted"""
pass
def incorporate(self, other_user):
"""Incorporate all enrollments and goals performed by the other user
If this user is not enrolled in a given experiment, the results for the
other user are incorporated. For experiments this user is already
enrolled in the results of the other user are discarded.
This takes a relatively large amount of time for each experiment the other
user is enrolled in."""
for enrollment in other_user._get_all_enrollments():
if not self._get_enrollment(enrollment.experiment):
self._set_enrollment(enrollment.experiment, enrollment.alternative, enrollment.enrollment_date, enrollment.last_seen)
goals = self.experiment_counter.participant_goal_frequencies(enrollment.experiment, enrollment.alternative, other_user._participant_identifier())
for goal_name, count in goals:
self.experiment_counter.increment_goal_count(enrollment.experiment, enrollment.alternative, goal_name, self._participant_identifier(), count)
other_user._cancel_enrollment(enrollment.experiment)
def visit(self):
"""Record that the user has visited the site for the purposes of retention tracking"""
for enrollment in self._get_all_enrollments():
if enrollment.experiment.is_displaying_alternatives():
if not enrollment.last_seen or now() - enrollment.last_seen >= timedelta(1):
self._experiment_goal(enrollment.experiment, enrollment.alternative, conf.VISIT_COUNT_GOAL, 1)
self._set_last_seen(enrollment.experiment, now())
def _get_enrollment(self, experiment):
"""Get the name of the alternative this user is enrolled in for the specified experiment
`experiment` is an instance of Experiment. If the user is not currently enrolled returns None."""
raise NotImplementedError
def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None):
"""Explicitly set the alternative the user is enrolled in for the specified experiment.
This allows you to change a user between alternatives. The user and goal counts for the new
alternative will be increment, but those for the old one will not be decremented."""
raise NotImplementedError
def is_enrolled(self, experiment_name, alternative, request):
"""Enroll this user in the experiment if they are not already part of it. Returns the selected alternative"""
"""Test if the user is enrolled in the supplied alternative for the given experiment.
The supplied alternative will be added to the list of possible alternatives for the
experiment if it is not already there. If the user is not yet enrolled in the supplied
experiment they will be enrolled, and an alternative chosen at random."""
chosen_alternative = self.enroll(experiment_name, [alternative])
return alternative == chosen_alternative
def _participant_identifier(self):
"Unique identifier for this user in the counter store"
raise NotImplementedError
def _get_all_enrollments(self):
"Return experiment, alternative tuples for all experiments the user is enrolled in"
raise NotImplementedError
def _cancel_enrollment(self, experiment):
"Remove the enrollment and any goals the user has against this experiment"
raise NotImplementedError
def _experiment_goal(self, experiment, alternative, goal_name, count):
"Record a goal against a particular experiment and alternative"
raise NotImplementedError
def _set_last_seen(self, experiment, last_seen):
"Set the last time the user was seen associated with this experiment"
raise NotImplementedError
def _gargoyle_key(self):
return None
def get_result_context(request, experiment):
experiment_counter = ExperimentCounter()
try:
chi2_goals = experiment.relevant_chi2_goals.replace(" ", "").split(",")
except AttributeError:
chi2_goals = ['']
try:
mwu_goals = experiment.relevant_mwu_goals.replace(" ", "").split(",")
except AttributeError:
mwu_goals = ['']
relevant_goals = set(chi2_goals + mwu_goals)
alternatives = {}
for alternative_name in experiment.alternatives.keys():
alternatives[alternative_name] = experiment_counter.participant_count(
experiment, alternative_name)
alternatives = sorted(alternatives.items())
control_participants = experiment_counter.participant_count(
experiment, conf.CONTROL_GROUP)
results = {}
for goal in conf.ALL_GOALS:
show_mwu = goal in mwu_goals
alternatives_conversions = {}
control_conversions = experiment_counter.goal_count(
experiment, conf.CONTROL_GROUP, goal)
control_conversion_rate = rate(control_conversions,
control_participants)
if show_mwu:
mwu_histogram = {}
control_conversion_distribution = fixup_distribution(
experiment_counter.goal_distribution(experiment,
conf.CONTROL_GROUP, goal),
control_participants)
control_average_goal_actions = average_actions(
control_conversion_distribution)
mwu_histogram['control'] = control_conversion_distribution
else:
control_average_goal_actions = None
for alternative_name in experiment.alternatives.keys():
if not alternative_name == conf.CONTROL_GROUP:
alternative_conversions = experiment_counter.goal_count(
experiment, alternative_name, goal)
alternative_participants = experiment_counter.participant_count(
experiment, alternative_name)
alternative_conversion_rate = rate(alternative_conversions,
alternative_participants)
alternative_confidence = chi_squared_confidence(
alternative_participants, alternative_conversions,
control_participants, control_conversions)
if show_mwu:
alternative_conversion_distribution = fixup_distribution(
experiment_counter.goal_distribution(
experiment, alternative_name, goal),
alternative_participants)
alternative_average_goal_actions = average_actions(
alternative_conversion_distribution)
alternative_distribution_confidence = mann_whitney_confidence(
alternative_conversion_distribution,
control_conversion_distribution)
mwu_histogram[
alternative_name] = alternative_conversion_distribution
else:
alternative_average_goal_actions = None
alternative_distribution_confidence = None
alternative = {
'conversions':
alternative_conversions,
'conversion_rate':
alternative_conversion_rate,
'improvement':
improvement(alternative_conversion_rate,
control_conversion_rate),
'confidence':
alternative_confidence,
'average_goal_actions':
alternative_average_goal_actions,
'mann_whitney_confidence':
alternative_distribution_confidence,
}
alternatives_conversions[alternative_name] = alternative
control = {
'conversions': control_conversions,
'conversion_rate': control_conversion_rate,
'average_goal_actions': control_average_goal_actions,
}
results[goal] = {
"control":
control,
"alternatives":
sorted(alternatives_conversions.items()),
"relevant":
goal in relevant_goals or relevant_goals == {''},
"mwu":
goal in mwu_goals,
"mwu_histogram":
conversion_distributions_to_graph_table(mwu_histogram)
if show_mwu else None
}
return {
'experiment': experiment.to_dict(),
'alternatives': alternatives,
'control_participants': control_participants,
'results': results,
'column_count': len(alternatives_conversions) * 3 +
2, # Horrible coupling with template design
'user_alternative':
participant(request).get_alternative(experiment.name),
}
def setUp(self):
self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE)
self.experiment_counter = ExperimentCounter()
