def test_finished_clears_out_users_participation_from_their_session(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {"link_color": alternative_name} finished('link_color') assert session['split'] == {}
def test_know_if_it_is_the_control_of_an_experiment(self): experiment = Experiment(self.redis, 'basket_text', 'Basket', 'Cart') experiment.save() alternative = Alternative(self.redis, 'Basket', 'basket_text') assert alternative.is_control alternative = Alternative(self.redis, 'Cart', 'basket_text') assert not alternative.is_control
def test_saves_the_start_time_to_redis(self): experiment_start_time = datetime(2012, 3, 9, 22, 01, 34) (flexmock(Experiment).should_receive('_get_time').and_return( experiment_start_time)) experiment = Experiment(self.redis, 'basket_text', 'Basket', 'Cart') experiment.save() assert experiment.start_time == experiment_start_time
def test_increment_completed_count(self): experiment = Experiment(self.redis, 'basket_text', 'Basket', "Cart") experiment.save() alternative = Alternative(self.redis, 'Basket', 'basket_text') old_completed_count = alternative.participant_count alternative.increment_completion() assert alternative.completed_count == old_completed_count + 1
def test_displays_the_start_date(self): experiment_start_time = datetime(2011, 7, 7) (flexmock(Experiment).should_receive('_get_time').and_return( experiment_start_time)) Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') response = self.client.get('/split/') assert '2011-07-07' in response.get_data(as_text=True)
def test_finished_increments_completed_alternative_counter(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') alternative = Alternative(self.redis, alternative_name, 'link_color') previous_completion_count = alternative.completed_count finished('link_color') new_completion_count = alternative.completed_count assert new_completion_count == previous_completion_count + 1
def test_allow_you_to_specify_a_winner(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') experiment.winner = 'red' experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') assert experiment.winner.name == 'red'
def test_allow_you_to_specify_a_winner(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') experiment.winner = 'red' experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') assert experiment.winner.name == 'red'
def test_saves_the_start_time_to_redis(self): experiment_start_time = datetime(2012, 3, 9, 22, 01, 34) (flexmock(Experiment) .should_receive('_get_time') .and_return(experiment_start_time)) experiment = Experiment(self.redis, 'basket_text', 'Basket', 'Cart') experiment.save() assert experiment.start_time == experiment_start_time
def test_displays_the_start_date(self): experiment_start_time = datetime(2011, 7, 7) (flexmock(Experiment) .should_receive('_get_time') .and_return(experiment_start_time)) Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') response = self.client.get('/split/') assert '2011-07-07' in response.data
def test_finished_dont_incr_completed_twice_if_no_reset(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') finished('link_color', reset=False) finished('link_color', reset=False) alternative = Alternative(self.redis, alternative_name, 'link_color') completion_count = alternative.completed_count assert completion_count == 1
def test_conversions_return_conversion_rates_for_alternatives(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.conversion_rate == 0.0 finished('link_color') assert alternative.conversion_rate == 1.0
def test_finished_dont_clear_out_the_users_session_if_reset_is_false(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {"link_color": alternative_name} finished('link_color', reset=False) assert session['split'] == { "link_color": alternative_name, } assert session['split_finished'] == set(['link_color'])
def test_handles_not_having_a_start_time(self): experiment_start_time = datetime(2012, 3, 9, 22, 01, 34) (flexmock(Experiment).should_receive('_get_time').and_return( experiment_start_time)) experiment = Experiment(self.redis, 'basket_text', 'Basket', 'Cart') experiment.save() self.redis.hdel('experiment_start_times', experiment.name) assert experiment.start_time is None
def test_handles_not_having_a_start_time(self): experiment_start_time = datetime(2012, 3, 9, 22, 01, 34) (flexmock(Experiment) .should_receive('_get_time') .and_return(experiment_start_time)) experiment = Experiment(self.redis, 'basket_text', 'Basket', 'Cart') experiment.save() self.redis.hdel('experiment_start_times', experiment.name) assert experiment.start_time is None
def test_next_alternative_always_returns_the_winner_if_one_exists(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red', 'green') green = Alternative(self.redis, 'green', 'link_color') experiment.winner = 'green' assert experiment.next_alternative().name == 'green' green.increment_participation() experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red', 'green') assert experiment.next_alternative().name == 'green'
def test_next_alternative_always_returns_the_winner_if_one_exists(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red', 'green') green = Alternative(self.redis, 'green', 'link_color') experiment.winner = 'green' assert experiment.next_alternative().name == 'green' green.increment_participation() experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red', 'green') assert experiment.next_alternative().name == 'green'
def test_reset_an_experiment_if_loaded_with_different_alternatives(self): experiment = Experiment(self.redis, 'link_color', 'blue', 'red', 'green') experiment.save() blue = Alternative(self.redis, 'blue', 'link_color') blue.participant_count = 5 blue.save() same_experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'yellow', 'orange') alternative_names = [a.name for a in same_experiment.alternatives] assert alternative_names == ['blue', 'yellow', 'orange'] new_blue = Alternative(self.redis, 'blue', 'link_color') assert new_blue.participant_count == 0
def test_reset_an_experiment(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') red = Alternative(self.redis, 'red', 'link_color') blue = Alternative(self.redis, 'blue', 'link_color') red.participant_count = 5 blue.participant_count = 6 response = self.client.post('/split/link_color/reset') assert_redirects(response, '/split/') assert red.participant_count == 0 assert blue.participant_count == 0
def test_reset_an_experiment_if_loaded_with_different_alternatives(self): experiment = Experiment( self.redis, 'link_color', 'blue', 'red', 'green') experiment.save() blue = Alternative(self.redis, 'blue', 'link_color') blue.participant_count = 5 blue.save() same_experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'yellow', 'orange') alternative_names = [a.name for a in same_experiment.alternatives] assert alternative_names == ['blue', 'yellow', 'orange'] new_blue = Alternative(self.redis, 'blue', 'link_color') assert new_blue.participant_count == 0
def test_ab_test_increments_participation_counter_for_new_user(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') red = Alternative(self.redis, 'red', 'link_color') blue = Alternative(self.redis, 'blue', 'link_color') previous_red_count = red.participant_count previous_blue_count = blue.participant_count ab_test('link_color', 'blue', 'red') new_red_count = red.participant_count new_blue_count = blue.participant_count assert (new_red_count + new_blue_count == previous_red_count + previous_blue_count + 1)
def test_saves_the_version_of_the_experiment_to_the_session(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') experiment.reset() assert experiment.version == 1 alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color:1': alternative_name}
def test_saves_the_version_of_the_experiment_to_the_session(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') experiment.reset() assert experiment.version == 1 alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color:1': alternative_name}
def test_loads_the_experiment_even_if_the_version_is_not_0(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') experiment.reset() assert experiment.version == 1 alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color:1': alternative_name} return_alternative_name = ab_test('link_color', 'blue', 'red') assert return_alternative_name == alternative_name
def test_loads_the_experiment_even_if_the_version_is_not_0(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') experiment.reset() assert experiment.version == 1 alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color:1': alternative_name} return_alternative_name = ab_test('link_color', 'blue', 'red') assert return_alternative_name == alternative_name
def test_mark_a_non_existing_alternative_as_the_winner(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') assert experiment.winner is None response = self.client.post('/split/link_color', data={'alternative': 'foobar'}) assert_redirects(response, '/split/') assert experiment.winner is None
def test_finished_clears_test_session_when_version_is_greater_than_0(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') experiment.increment_version() alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {"link_color:1": alternative_name} finished('link_color') assert session['split'] == {}
def test_mark_a_non_existing_alternative_as_the_winner(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') assert experiment.winner is None response = self.client.post('/split/link_color', data={'alternative': 'foobar'}) assert_redirects(response, '/split/') assert experiment.winner is None
def test_finished_clears_test_session_when_version_is_greater_than_0(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') experiment.increment_version() alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {"link_color:1": alternative_name} finished('link_color') assert session['split'] == {}
def test_z_score_when_control_and_alternative_have_perfect_conversion(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') experiment.save() control = Alternative(self.redis, 'blue', 'link_color') control.completed_count = 10 control.participant_count = 10 alternative = Alternative(self.redis, 'red', 'link_color') alternative.completed_count = 8 alternative.participant_count = 8 assert alternative.z_score is None
def test_z_score_when_control_and_alternative_have_perfect_conversion( self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') experiment.save() control = Alternative(self.redis, 'blue', 'link_color') control.completed_count = 10 control.participant_count = 10 alternative = Alternative(self.redis, 'red', 'link_color') alternative.completed_count = 8 alternative.participant_count = 8 assert alternative.z_score is None
def test_reset_should_reset_the_winner(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red', 'green') green = Alternative(self.redis, 'green', 'link_color') experiment.winner = 'green' assert experiment.next_alternative().name == 'green' green.increment_participation() experiment.reset() assert experiment.winner is None
def test_reset_should_reset_the_winner(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red', 'green') green = Alternative(self.redis, 'green', 'link_color') experiment.winner = 'green' assert experiment.next_alternative().name == 'green' green.increment_participation() experiment.reset() assert experiment.winner is None
def test_only_counts_completion_of_users_on_the_current_version(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color': alternative_name} alternative = Alternative(self.redis, alternative_name, 'link_color') experiment.reset() assert experiment.version == 1 finished('link_color') alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.completed_count == 0
def test_only_counts_completion_of_users_on_the_current_version(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color': alternative_name} alternative = Alternative(self.redis, alternative_name, 'link_color') experiment.reset() assert experiment.version == 1 finished('link_color') alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.completed_count == 0
def test_reset_should_reset_all_alternatives(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red', 'green') green = Alternative(self.redis, 'green', 'link_color') experiment.winner = 'green' assert experiment.next_alternative().name == 'green' green.increment_participation() experiment.reset() reset_green = Alternative(self.redis, 'green', 'link_color') assert reset_green.participant_count == 0 assert reset_green.completed_count == 0
def test_reset_should_reset_all_alternatives(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red', 'green') green = Alternative(self.redis, 'green', 'link_color') experiment.winner = 'green' assert experiment.next_alternative().name == 'green' green.increment_participation() experiment.reset() reset_green = Alternative(self.redis, 'green', 'link_color') assert reset_green.participant_count == 0 assert reset_green.completed_count == 0
def test_cleans_up_old_versions_of_experiments_from_the_session(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color': alternative_name} alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 1 experiment.reset() assert experiment.version == 1 alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 0 new_alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color:1': new_alternative_name}
def test_z_score(self): Experiment.find_or_create(self.redis, 'Treatment', 'Control', 'Treatment A', 'Treatment B', 'Treatment C') control = Alternative(self.redis, 'Control', 'Treatment') control.participant_count = 182 control.completed_count = 35 treatment_a = Alternative(self.redis, 'Treatment A', 'Treatment') treatment_a.participant_count = 180 treatment_a.completed_count = 45 treatment_b = Alternative(self.redis, 'Treatment B', 'Treatment') treatment_b.participant_count = 189 treatment_b.completed_count = 28 treatment_c = Alternative(self.redis, 'Treatment C', 'Treatment') treatment_c.participant_count = 188 treatment_c.completed_count = 61 assert control.z_score is None assert round(treatment_a.z_score, 2) == 1.33 assert round(treatment_b.z_score, 2) == -1.13 assert round(treatment_c.z_score, 2) == 2.94
def test_cleans_up_old_versions_of_experiments_from_the_session(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color': alternative_name} alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 1 experiment.reset() assert experiment.version == 1 alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 0 new_alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color:1': new_alternative_name}
def test_resets_users_session_on_an_older_version_of_the_experiment(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color': alternative_name} alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 1 experiment.reset() assert experiment.version == 1 alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 0 new_alternative_name = ab_test('link_color', 'blue', 'red') assert session['split']['link_color:1'] == new_alternative_name new_alternative = Alternative( self.redis, new_alternative_name, 'link_color') assert new_alternative.participant_count == 1
def test_resets_users_session_on_an_older_version_of_the_experiment(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert session['split'] == {'link_color': alternative_name} alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 1 experiment.reset() assert experiment.version == 1 alternative = Alternative(self.redis, alternative_name, 'link_color') assert alternative.participant_count == 0 new_alternative_name = ab_test('link_color', 'blue', 'red') assert session['split']['link_color:1'] == new_alternative_name new_alternative = Alternative(self.redis, new_alternative_name, 'link_color') assert new_alternative.participant_count == 1
def test_ab_test_always_returns_the_winner_if_one_is_present(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') experiment.winner = "orange" assert ab_test('link_color', 'blue', 'red') == 'orange'
def test_ab_test_returns_the_given_alternative_for_an_existing_user(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative = ab_test('link_color', 'blue', 'red') repeat_alternative = ab_test('link_color', 'blue', 'red') assert alternative == repeat_alternative
def test_uses_version_zero_if_no_version_is_present(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert experiment.version == 0 assert session['split'] == {'link_color': alternative_name}
def test_ab_test_always_returns_the_winner_if_one_is_present(self): experiment = Experiment.find_or_create( self.redis, 'link_color', 'blue', 'red') experiment.winner = "orange" assert ab_test('link_color', 'blue', 'red') == 'orange'
def test_uses_version_zero_if_no_version_is_present(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') assert experiment.version == 0 assert session['split'] == {'link_color': alternative_name}
def test_ab_test_return_the_control(self): experiment = Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') alternative = ab_test('link_color', 'blue', 'red') assert alternative == experiment.control.name
def test_delete_an_experiment(self): Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red') response = self.client.post('/split/link_color/delete') assert_redirects(response, '/split/') assert Experiment.find(self.redis, 'link_color') is None