class ConfirmHumanTestCase(TestCase): def setUp(self): self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE) self.experiment_counter = ExperimentCounter() self.experiment_user = participant(session=DatabaseSession()) self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative']) self.experiment_user.goal('my_goal') self.redis = get_redis_client() def tearDown(self): self.experiment_counter.delete(self.experiment) def test_confirm_human_updates_experiment(self): self.assertTrue(self.redis.exists(self.experiment_user._redis_goals_key)) self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) self.experiment_user.confirm_human() self.assertFalse(self.redis.exists(self.experiment_user._redis_goals_key)) self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) def test_confirm_human_called_twice(self): """ Ensuring that counters aren't incremented twice """ self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) self.experiment_user.confirm_human() self.experiment_user.confirm_human() self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) def test_confirm_human_sets_session(self): self.assertFalse(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) self.experiment_user.confirm_human() self.assertTrue(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) def test_session_already_confirmed(self): """ Testing that confirm_human works even if code outside of django-experiments updates the key """ self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True self.experiment_user.confirm_human() self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
class ConfirmHumanTestCase(TestCase): def setUp(self): self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE) self.experiment_counter = ExperimentCounter() self.experiment_user = participant(session=DatabaseSession()) self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative']) self.experiment_user.goal('my_goal') def tearDown(self): self.experiment_counter.delete(self.experiment) def test_confirm_human_updates_experiment(self): self.assertIn('experiments_goals', self.experiment_user.session) self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) self.experiment_user.confirm_human() self.assertNotIn('experiments_goals', self.experiment_user.session) self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) def test_confirm_human_called_twice(self): """ Ensuring that counters aren't incremented twice """ self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) self.experiment_user.confirm_human() self.experiment_user.confirm_human() self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) def test_confirm_human_sets_session(self): self.assertFalse(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) self.experiment_user.confirm_human() self.assertTrue(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) def test_session_already_confirmed(self): """ Testing that confirm_human works even if code outside of django-experiments updates the key """ self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True self.experiment_user.confirm_human() self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1)
class IncorporateTestCase(TestCase): def setUp(self): self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment_counter = ExperimentCounter() User = get_user_model() self.user = User.objects.create(username='******') self.user.is_confirmed_human = True request_factory = RequestFactory() self.request = request_factory.get('/') self.request.session = DatabaseSession() participant(self.request).confirm_human() def tearDown(self): self.experiment_counter.delete(self.experiment) def _login(self): self.request.user = self.user transfer_enrollments_to_user(None, self.request, self.user) def test_visit_incorporate(self): alternative = participant(self.request).enroll(self.experiment.name, ['alternative']) ExperimentsRetentionMiddleware().process_response( self.request, HttpResponse()) self.assertEqual( dict( self.experiment_counter.participant_goal_frequencies( self.experiment, alternative, participant(self.request)._participant_identifier()))[ conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1) self.assertFalse( Enrollment.objects.filter(user__isnull=False).exists()) self._login() self.assertTrue(Enrollment.objects.filter(user__isnull=False).exists()) self.assertIsNotNone(Enrollment.objects.all()[0].last_seen) self.assertEqual( dict( self.experiment_counter.participant_goal_frequencies( self.experiment, alternative, participant(self.request)._participant_identifier()))[ conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1) self.assertEqual( self.experiment_counter.goal_count( self.experiment, alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1) self.assertEqual( self.experiment_counter.participant_count(self.experiment, alternative), 1)
def test_reset_all(self): experiment = Experiment.objects.create(name='reset_test') other_experiment = Experiment.objects.create(name='reset_test_other') experiment_counter = ExperimentCounter() for exp in [experiment, other_experiment]: experiment_counter.increment_participant_count(exp, 'alt', 'fred') experiment_counter.increment_participant_count(exp, 'alt', 'fred') experiment_counter.increment_participant_count(exp, 'alt', 'fred') experiment_counter.increment_goal_count(exp, 'alt', 'goal1', 'fred') experiment_counter.increment_goal_count(exp, 'alt', 'goal2', 'fred') experiment_counter.increment_participant_count( exp, 'control', 'barney') experiment_counter.increment_participant_count( exp, 'control', 'wilma') experiment_counter.increment_participant_count( exp, 'control', 'betty') experiment_counter.increment_goal_count(exp, 'control', 'goal1', 'betty') self.counters.reset_prefix(experiment.name) self.assertEqual( experiment_counter.participant_count(experiment, 'alt'), 0) self.assertEqual( experiment_counter.participant_count(experiment, 'control'), 0) self.assertEqual( experiment_counter.goal_count(experiment, 'control', 'goal1'), 0) self.assertEqual( experiment_counter.participant_count(other_experiment, 'alt'), 1) self.assertEqual( experiment_counter.participant_count(other_experiment, 'control'), 3) self.assertEqual( experiment_counter.goal_count(other_experiment, 'control', 'goal1'), 1)
class IncorporateTestCase(TestCase): def setUp(self): self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment_counter = ExperimentCounter() User = get_user_model() self.user = User.objects.create(username='******') self.user.is_confirmed_human = True request_factory = RequestFactory() self.request = request_factory.get('/') self.request.session = DatabaseSession() participant(self.request).confirm_human() def tearDown(self): self.experiment_counter.delete(self.experiment) def _login(self): self.request.user = self.user transfer_enrollments_to_user(None, self.request, self.user) def test_visit_incorporate(self): alternative = participant(self.request).enroll(self.experiment.name, ['alternative']) ExperimentsRetentionMiddleware().process_response(self.request, HttpResponse()) self.assertEqual( dict(self.experiment_counter.participant_goal_frequencies(self.experiment, alternative, participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1 ) self.assertFalse(Enrollment.objects.all().exists()) self._login() self.assertTrue(Enrollment.objects.all().exists()) self.assertIsNotNone(Enrollment.objects.all()[0].last_seen) self.assertEqual( dict(self.experiment_counter.participant_goal_frequencies(self.experiment, alternative, participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1 ) self.assertEqual(self.experiment_counter.goal_count(self.experiment, alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1) self.assertEqual(self.experiment_counter.participant_count(self.experiment, alternative), 1)
def get_result_context(request, experiment): experiment_counter = ExperimentCounter() try: chi2_goals = experiment.relevant_chi2_goals.replace(" ", "").split(",") except AttributeError: chi2_goals = [''] try: mwu_goals = experiment.relevant_mwu_goals.replace(" ", "").split(",") except AttributeError: mwu_goals = [''] relevant_goals = set(chi2_goals + mwu_goals) alternatives = {} for alternative_name in experiment.alternatives.keys(): alternatives[alternative_name] = experiment_counter.participant_count( experiment, alternative_name) alternatives = sorted(alternatives.items()) control_participants = experiment_counter.participant_count( experiment, conf.CONTROL_GROUP) results = {} for goal in conf.ALL_GOALS: show_mwu = goal in mwu_goals alternatives_conversions = {} control_conversions = experiment_counter.goal_count( experiment, conf.CONTROL_GROUP, goal) control_conversion_rate = rate(control_conversions, control_participants) if show_mwu: mwu_histogram = {} control_conversion_distribution = fixup_distribution( experiment_counter.goal_distribution(experiment, conf.CONTROL_GROUP, goal), control_participants) control_average_goal_actions = average_actions( control_conversion_distribution) mwu_histogram['control'] = control_conversion_distribution else: control_average_goal_actions = None for alternative_name in experiment.alternatives.keys(): if not alternative_name == conf.CONTROL_GROUP: alternative_conversions = experiment_counter.goal_count( experiment, alternative_name, goal) alternative_participants = experiment_counter.participant_count( experiment, alternative_name) alternative_conversion_rate = rate(alternative_conversions, alternative_participants) alternative_confidence = chi_squared_confidence( alternative_participants, alternative_conversions, control_participants, control_conversions) if show_mwu: alternative_conversion_distribution = fixup_distribution( experiment_counter.goal_distribution( experiment, alternative_name, goal), alternative_participants) alternative_average_goal_actions = average_actions( alternative_conversion_distribution) alternative_distribution_confidence = mann_whitney_confidence( alternative_conversion_distribution, control_conversion_distribution) mwu_histogram[ alternative_name] = alternative_conversion_distribution else: alternative_average_goal_actions = None alternative_distribution_confidence = None alternative = { 'conversions': alternative_conversions, 'conversion_rate': alternative_conversion_rate, 'improvement': improvement(alternative_conversion_rate, control_conversion_rate), 'confidence': alternative_confidence, 'average_goal_actions': alternative_average_goal_actions, 'mann_whitney_confidence': alternative_distribution_confidence, } alternatives_conversions[alternative_name] = alternative control = { 'conversions': control_conversions, 'conversion_rate': control_conversion_rate, 'average_goal_actions': control_average_goal_actions, } results[goal] = { "control": control, "alternatives": sorted(alternatives_conversions.items()), "relevant": goal in relevant_goals or relevant_goals == {''}, "mwu": goal in mwu_goals, "mwu_histogram": conversion_distributions_to_graph_table(mwu_histogram) if show_mwu else None } return { 'experiment': experiment.to_dict(), 'alternatives': alternatives, 'control_participants': control_participants, 'results': results, 'column_count': len(alternatives_conversions) * 3 + 2, # Horrible coupling with template design 'user_alternative': participant(request).get_alternative(experiment.name), }
def get_result_context(request, experiment): experiment_counter = ExperimentCounter() try: chi2_goals = experiment.relevant_chi2_goals.replace(" ", "").split(",") except AttributeError: chi2_goals = [u''] try: mwu_goals = experiment.relevant_mwu_goals.replace(" ", "").split(",") except AttributeError: mwu_goals = [u''] relevant_goals = set(chi2_goals + mwu_goals) alternatives = {} for alternative_name in experiment.alternatives.keys(): alternatives[alternative_name] = experiment_counter.participant_count(experiment, alternative_name) alternatives = sorted(alternatives.items()) control_participants = experiment_counter.participant_count(experiment, conf.CONTROL_GROUP) results = {} for goal in conf.ALL_GOALS: show_mwu = goal in mwu_goals alternatives_conversions = {} control_conversions = experiment_counter.goal_count(experiment, conf.CONTROL_GROUP, goal) control_conversion_rate = rate(control_conversions, control_participants) if show_mwu: mwu_histogram = {} control_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, conf.CONTROL_GROUP, goal), control_participants) control_average_goal_actions = average_actions(control_conversion_distribution) mwu_histogram['control'] = control_conversion_distribution else: control_average_goal_actions = None for alternative_name in experiment.alternatives.keys(): if not alternative_name == conf.CONTROL_GROUP: alternative_conversions = experiment_counter.goal_count(experiment, alternative_name, goal) alternative_participants = experiment_counter.participant_count(experiment, alternative_name) alternative_conversion_rate = rate(alternative_conversions, alternative_participants) alternative_confidence = chi_squared_confidence(alternative_participants, alternative_conversions, control_participants, control_conversions) if show_mwu: alternative_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, alternative_name, goal), alternative_participants) alternative_average_goal_actions = average_actions(alternative_conversion_distribution) alternative_distribution_confidence = mann_whitney_confidence(alternative_conversion_distribution, control_conversion_distribution) mwu_histogram[alternative_name] = alternative_conversion_distribution else: alternative_average_goal_actions = None alternative_distribution_confidence = None alternative = { 'conversions': alternative_conversions, 'conversion_rate': alternative_conversion_rate, 'improvement': improvement(alternative_conversion_rate, control_conversion_rate), 'confidence': alternative_confidence, 'average_goal_actions': alternative_average_goal_actions, 'mann_whitney_confidence': alternative_distribution_confidence, } alternatives_conversions[alternative_name] = alternative control = { 'conversions': control_conversions, 'conversion_rate': control_conversion_rate, 'average_goal_actions': control_average_goal_actions, } results[goal] = { "control": control, "alternatives": sorted(alternatives_conversions.items()), "relevant": goal in relevant_goals or relevant_goals == {u''}, "mwu": goal in mwu_goals, "mwu_histogram": conversion_distributions_to_graph_table(mwu_histogram) if show_mwu else None } return { 'experiment': experiment.to_dict(), 'alternatives': alternatives, 'control_participants': control_participants, 'results': results, 'column_count': len(alternatives_conversions) * 3 + 2, # Horrible coupling with template design 'user_alternative': participant(request).get_alternative(experiment.name), }
class WebUserTests(object): def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() self.experiment_counter = ExperimentCounter() def tearDown(self): self.experiment_counter.delete(self.experiment) def test_enrollment_initially_control(self): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control") def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set") def test_record_goal_increments_counts(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count") def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution") def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): thetime = timezone.now() with patch('experiments.utils.now', return_value=thetime): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") with patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)): experiment_user.visit() self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted") def test_visit_twice_increases_once(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") def test_user_force_enrolls(self): experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2') self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2') def test_user_does_not_force_enroll_to_new_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives) alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice(list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) def test_second_force_enroll_does_not_change_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1') alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice(list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
class WebUserTests(object): def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get("/") self.request.session = DatabaseSession() self.experiment_counter = ExperimentCounter() def tearDown(self): self.experiment_counter.delete(self.experiment) def confirm_human(self, experiment_user): pass def participants(self, alternative): return self.experiment_counter.participant_count(self.experiment, alternative) def enrollment_initially_none(self,): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), None) def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE) def test_record_goal_increments_counts(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1) def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1) def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1} ) def test_visit_twice_increases_once(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1} )
class WebUserTests(object): def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() self.experiment_counter = ExperimentCounter() def tearDown(self): self.experiment_counter.delete(self.experiment) def test_enrollment_initially_control(self): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control") def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set") def test_record_goal_increments_counts(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count") def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly") self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution") def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual( self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): thetime = timezone.now() with mock.patch('experiments.utils.now', return_value=thetime): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted") self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") with mock.patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)): experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted") self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted") def test_visit_twice_increases_once(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted") self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") def test_user_force_enrolls(self): experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2') self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2') def test_user_does_not_force_enroll_to_new_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives) alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice( list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) def test_second_force_enroll_does_not_change_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1') alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice( list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
class WebUserTests(object): def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() self.experiment_counter = ExperimentCounter() def tearDown(self): self.experiment_counter.delete(self.experiment) def confirm_human(self, experiment_user): pass def participants(self, alternative): return self.experiment_counter.participant_count( self.experiment, alternative) def enrollment_initially_none(self, ): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), None) def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE) def test_record_goal_increments_counts(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1) def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1) def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}) def test_visit_twice_increases_once(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1})