def test_displays_the_start_date(self):
experiment_start_time = datetime(2011, 7, 7)
(flexmock(Experiment).should_receive('_get_time').and_return(
experiment_start_time))
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
response = self.client.get('/split/')
assert '2011-07-07' in response.get_data(as_text=True)
def test_finished_clears_out_users_participation_from_their_session(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color": alternative_name}
finished('link_color')
assert session['split'] == {}
def test_finished_clears_out_users_participation_from_their_session(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color": alternative_name}
finished('link_color')
assert session['split'] == {}
def test_finished_increments_completed_alternative_counter(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
previous_completion_count = alternative.completed_count
finished('link_color')
new_completion_count = alternative.completed_count
assert new_completion_count == previous_completion_count + 1
def test_displays_the_start_date(self):
experiment_start_time = datetime(2011, 7, 7)
(flexmock(Experiment)
.should_receive('_get_time')
.and_return(experiment_start_time))
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
response = self.client.get('/split/')
assert '2011-07-07' in response.data
def test_allow_you_to_specify_a_winner(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.winner = 'red'
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
assert experiment.winner.name == 'red'
def test_allow_you_to_specify_a_winner(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.winner = 'red'
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
assert experiment.winner.name == 'red'
def test_finished_increments_completed_alternative_counter(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
previous_completion_count = alternative.completed_count
finished('link_color')
new_completion_count = alternative.completed_count
assert new_completion_count == previous_completion_count + 1
def test_finished_dont_incr_completed_twice_if_no_reset(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
finished('link_color', reset=False)
finished('link_color', reset=False)
alternative = Alternative(self.redis, alternative_name, 'link_color')
completion_count = alternative.completed_count
assert completion_count == 1
def test_conversions_return_conversion_rates_for_alternatives(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.conversion_rate == 0.0
finished('link_color')
assert alternative.conversion_rate == 1.0
def test_finished_dont_clear_out_the_users_session_if_reset_is_false(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color": alternative_name}
finished('link_color', reset=False)
assert session['split'] == {
"link_color": alternative_name,
}
assert session['split_finished'] == set(['link_color'])
def test_conversions_return_conversion_rates_for_alternatives(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.conversion_rate == 0.0
finished('link_color')
assert alternative.conversion_rate == 1.0
def test_finished_dont_incr_completed_twice_if_no_reset(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
finished('link_color', reset=False)
finished('link_color', reset=False)
alternative = Alternative(self.redis, alternative_name, 'link_color')
completion_count = alternative.completed_count
assert completion_count == 1
def test_finished_dont_clear_out_the_users_session_if_reset_is_false(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color": alternative_name}
finished('link_color', reset=False)
assert session['split'] == {
"link_color": alternative_name,
}
assert session['split_finished'] == set(['link_color'])
def test_next_alternative_always_returns_the_winner_if_one_exists(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
assert experiment.next_alternative().name == 'green'
def test_next_alternative_always_returns_the_winner_if_one_exists(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red', 'green')
assert experiment.next_alternative().name == 'green'
def test_reset_an_experiment(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
red.participant_count = 5
blue.participant_count = 6
response = self.client.post('/split/link_color/reset')
assert_redirects(response, '/split/')
assert red.participant_count == 0
assert blue.participant_count == 0
def test_reset_an_experiment(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
red.participant_count = 5
blue.participant_count = 6
response = self.client.post('/split/link_color/reset')
assert_redirects(response, '/split/')
assert red.participant_count == 0
assert blue.participant_count == 0
def test_ab_test_increments_participation_counter_for_new_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
previous_red_count = red.participant_count
previous_blue_count = blue.participant_count
ab_test('link_color', 'blue', 'red')
new_red_count = red.participant_count
new_blue_count = blue.participant_count
assert (new_red_count + new_blue_count ==
previous_red_count + previous_blue_count + 1)
def test_ab_test_increments_participation_counter_for_new_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
previous_red_count = red.participant_count
previous_blue_count = blue.participant_count
ab_test('link_color', 'blue', 'red')
new_red_count = red.participant_count
new_blue_count = blue.participant_count
assert (new_red_count + new_blue_count == previous_red_count +
previous_blue_count + 1)
def test_saves_the_version_of_the_experiment_to_the_session(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.reset()
assert experiment.version == 1
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': alternative_name}
def test_saves_the_version_of_the_experiment_to_the_session(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.reset()
assert experiment.version == 1
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': alternative_name}
def test_loads_the_experiment_even_if_the_version_is_not_0(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.reset()
assert experiment.version == 1
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': alternative_name}
return_alternative_name = ab_test('link_color', 'blue', 'red')
assert return_alternative_name == alternative_name
def test_loads_the_experiment_even_if_the_version_is_not_0(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.reset()
assert experiment.version == 1
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': alternative_name}
return_alternative_name = ab_test('link_color', 'blue', 'red')
assert return_alternative_name == alternative_name
def test_finished_clears_test_session_when_version_is_greater_than_0(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.increment_version()
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color:1": alternative_name}
finished('link_color')
assert session['split'] == {}
def test_finished_clears_test_session_when_version_is_greater_than_0(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.increment_version()
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color:1": alternative_name}
finished('link_color')
assert session['split'] == {}
def test_mark_a_non_existing_alternative_as_the_winner(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
assert experiment.winner is None
response = self.client.post('/split/link_color',
data={'alternative': 'foobar'})
assert_redirects(response, '/split/')
assert experiment.winner is None
def test_mark_a_non_existing_alternative_as_the_winner(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
assert experiment.winner is None
response = self.client.post('/split/link_color',
data={'alternative': 'foobar'})
assert_redirects(response, '/split/')
assert experiment.winner is None
def test_z_score_when_control_and_alternative_have_perfect_conversion(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.save()
control = Alternative(self.redis, 'blue', 'link_color')
control.completed_count = 10
control.participant_count = 10
alternative = Alternative(self.redis, 'red', 'link_color')
alternative.completed_count = 8
alternative.participant_count = 8
assert alternative.z_score is None
def test_z_score_when_control_and_alternative_have_perfect_conversion(
self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.save()
control = Alternative(self.redis, 'blue', 'link_color')
control.completed_count = 10
control.participant_count = 10
alternative = Alternative(self.redis, 'red', 'link_color')
alternative.completed_count = 8
alternative.participant_count = 8
assert alternative.z_score is None
def test_reset_should_reset_the_winner(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment.reset()
assert experiment.winner is None
def test_reset_should_reset_the_winner(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment.reset()
assert experiment.winner is None
def test_reset_an_experiment_if_loaded_with_different_alternatives(self):
experiment = Experiment(self.redis, 'link_color', 'blue', 'red',
'green')
experiment.save()
blue = Alternative(self.redis, 'blue', 'link_color')
blue.participant_count = 5
blue.save()
same_experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'yellow', 'orange')
alternative_names = [a.name for a in same_experiment.alternatives]
assert alternative_names == ['blue', 'yellow', 'orange']
new_blue = Alternative(self.redis, 'blue', 'link_color')
assert new_blue.participant_count == 0
def test_only_counts_completion_of_users_on_the_current_version(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
experiment.reset()
assert experiment.version == 1
finished('link_color')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.completed_count == 0
def test_reset_an_experiment_if_loaded_with_different_alternatives(self):
experiment = Experiment(
self.redis, 'link_color', 'blue', 'red', 'green')
experiment.save()
blue = Alternative(self.redis, 'blue', 'link_color')
blue.participant_count = 5
blue.save()
same_experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'yellow', 'orange')
alternative_names = [a.name for a in same_experiment.alternatives]
assert alternative_names == ['blue', 'yellow', 'orange']
new_blue = Alternative(self.redis, 'blue', 'link_color')
assert new_blue.participant_count == 0
def test_only_counts_completion_of_users_on_the_current_version(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
experiment.reset()
assert experiment.version == 1
finished('link_color')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.completed_count == 0
def test_reset_should_reset_all_alternatives(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment.reset()
reset_green = Alternative(self.redis, 'green', 'link_color')
assert reset_green.participant_count == 0
assert reset_green.completed_count == 0
def test_reset_should_reset_all_alternatives(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment.reset()
reset_green = Alternative(self.redis, 'green', 'link_color')
assert reset_green.participant_count == 0
assert reset_green.completed_count == 0
def test_cleans_up_old_versions_of_experiments_from_the_session(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': new_alternative_name}
def test_z_score(self):
Experiment.find_or_create(self.redis, 'Treatment', 'Control',
'Treatment A', 'Treatment B', 'Treatment C')
control = Alternative(self.redis, 'Control', 'Treatment')
control.participant_count = 182
control.completed_count = 35
treatment_a = Alternative(self.redis, 'Treatment A', 'Treatment')
treatment_a.participant_count = 180
treatment_a.completed_count = 45
treatment_b = Alternative(self.redis, 'Treatment B', 'Treatment')
treatment_b.participant_count = 189
treatment_b.completed_count = 28
treatment_c = Alternative(self.redis, 'Treatment C', 'Treatment')
treatment_c.participant_count = 188
treatment_c.completed_count = 61
assert control.z_score is None
assert round(treatment_a.z_score, 2) == 1.33
assert round(treatment_b.z_score, 2) == -1.13
assert round(treatment_c.z_score, 2) == 2.94
def test_z_score(self):
Experiment.find_or_create(self.redis, 'Treatment',
'Control', 'Treatment A', 'Treatment B', 'Treatment C')
control = Alternative(self.redis, 'Control', 'Treatment')
control.participant_count = 182
control.completed_count = 35
treatment_a = Alternative(self.redis, 'Treatment A', 'Treatment')
treatment_a.participant_count = 180
treatment_a.completed_count = 45
treatment_b = Alternative(self.redis, 'Treatment B', 'Treatment')
treatment_b.participant_count = 189
treatment_b.completed_count = 28
treatment_c = Alternative(self.redis, 'Treatment C', 'Treatment')
treatment_c.participant_count = 188
treatment_c.completed_count = 61
assert control.z_score is None
assert round(treatment_a.z_score, 2) == 1.33
assert round(treatment_b.z_score, 2) == -1.13
assert round(treatment_c.z_score, 2) == 2.94
def test_cleans_up_old_versions_of_experiments_from_the_session(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': new_alternative_name}
def test_resets_users_session_on_an_older_version_of_the_experiment(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split']['link_color:1'] == new_alternative_name
new_alternative = Alternative(self.redis, new_alternative_name,
'link_color')
assert new_alternative.participant_count == 1
def test_resets_users_session_on_an_older_version_of_the_experiment(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split']['link_color:1'] == new_alternative_name
new_alternative = Alternative(
self.redis, new_alternative_name, 'link_color')
assert new_alternative.participant_count == 1
def test_z_score_is_none_for_the_control(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
assert experiment.control.z_score is None
def test_z_score_is_none_when_alternative_has_no_participations(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.save()
alternative = Alternative(self.redis, 'red', 'link_color')
assert alternative.z_score is None
def test_ab_test_return_the_control(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
assert alternative == experiment.control.name
def test_delete_an_experiment(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
response = self.client.post('/split/link_color/delete')
assert_redirects(response, '/split/')
assert Experiment.find(self.redis, 'link_color') is None
def test_ab_test_always_returns_the_winner_if_one_is_present(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.winner = "orange"
assert ab_test('link_color', 'blue', 'red') == 'orange'
def test_uses_version_zero_if_no_version_is_present(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert experiment.version == 0
assert session['split'] == {'link_color': alternative_name}
def test_ab_test_returns_the_given_alternative_for_an_existing_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
repeat_alternative = ab_test('link_color', 'blue', 'red')
assert alternative == repeat_alternative
def test_ab_test_always_returns_the_winner_if_one_is_present(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.winner = "orange"
assert ab_test('link_color', 'blue', 'red') == 'orange'
def test_delete_an_experiment(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
response = self.client.post('/split/link_color/delete')
assert_redirects(response, '/split/')
assert Experiment.find(self.redis, 'link_color') is None
def test_reset_should_increment_the_version(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red', 'green')
assert experiment.version == 0
experiment.reset()
assert experiment.version == 1
def test_z_score_is_none_for_the_control(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
assert experiment.control.z_score is None
def test_reset_should_increment_the_version(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
assert experiment.version == 0
experiment.reset()
assert experiment.version == 1
def test_ab_test_returns_the_given_alternative_for_an_existing_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
repeat_alternative = ab_test('link_color', 'blue', 'red')
assert alternative == repeat_alternative
def test_ab_test_return_the_control(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
assert alternative == experiment.control.name
def test_uses_version_zero_if_no_version_is_present(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert experiment.version == 0
assert session['split'] == {'link_color': alternative_name}
def test_z_score_is_none_when_alternative_has_no_participations(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.save()
alternative = Alternative(self.redis, 'red', 'link_color')
assert alternative.z_score is None
