def test_can_be_reset(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
alternative.participant_count = 10
alternative.completed_count = 4
alternative.reset()
assert alternative.participant_count == 0
assert alternative.completed_count == 0
def test_increment_completed_count(self):
experiment = Experiment(self.redis, 'basket_text', 'Basket', "Cart")
experiment.save()
alternative = Alternative(self.redis, 'Basket', 'basket_text')
old_completed_count = alternative.participant_count
alternative.increment_completion()
assert alternative.completed_count == old_completed_count + 1
def test_next_alternative_always_returns_the_winner_if_one_exists(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
assert experiment.next_alternative().name == 'green'
def test_reset_should_reset_the_winner(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment.reset()
assert experiment.winner is None
def test_reset_an_experiment(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
red.participant_count = 5
blue.participant_count = 6
response = self.client.post('/split/link_color/reset')
assert_redirects(response, '/split/')
assert red.participant_count == 0
assert blue.participant_count == 0
def test_reset_an_experiment_if_loaded_with_different_alternatives(self):
experiment = Experiment(
self.redis, 'link_color', 'blue', 'red', 'green')
experiment.save()
blue = Alternative(self.redis, 'blue', 'link_color')
blue.participant_count = 5
blue.save()
same_experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'yellow', 'orange')
alternative_names = [a.name for a in same_experiment.alternatives]
assert alternative_names == ['blue', 'yellow', 'orange']
new_blue = Alternative(self.redis, 'blue', 'link_color')
assert new_blue.participant_count == 0
def test_reset_should_reset_all_alternatives(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red', 'green')
green = Alternative(self.redis, 'green', 'link_color')
experiment.winner = 'green'
assert experiment.next_alternative().name == 'green'
green.increment_participation()
experiment.reset()
reset_green = Alternative(self.redis, 'green', 'link_color')
assert reset_green.participant_count == 0
assert reset_green.completed_count == 0
def test_cleans_up_old_versions_of_experiments_from_the_session(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': new_alternative_name}
def test_ab_test_increments_participation_counter_for_new_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
previous_red_count = red.participant_count
previous_blue_count = blue.participant_count
ab_test('link_color', 'blue', 'red')
new_red_count = red.participant_count
new_blue_count = blue.participant_count
assert (new_red_count + new_blue_count == previous_red_count +
previous_blue_count + 1)
def test_can_participate_in_many_experiments_with_allow_multiple_experiments(self):
self.app.config['SPLIT_ALLOW_MULTIPLE_EXPERIMENTS'] = True
link_color = ab_test('link_color', 'blue', 'red')
button_size = ab_test('button_size', 'small', 'big')
assert _get_session()['button_size'] == button_size
button_size_alt = Alternative(self.redis, button_size, 'button_size')
assert button_size_alt.participant_count == 1
def test_finished_increments_completed_alternative_counter(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
previous_completion_count = alternative.completed_count
finished('link_color')
new_completion_count = alternative.completed_count
assert new_completion_count == previous_completion_count + 1
def test_resets_users_session_on_an_older_version_of_the_experiment(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split']['link_color:1'] == new_alternative_name
new_alternative = Alternative(self.redis, new_alternative_name,
'link_color')
assert new_alternative.participant_count == 1
def test_conversions_return_conversion_rates_for_alternatives(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.conversion_rate == 0.0
finished('link_color')
assert alternative.conversion_rate == 1.0
def test_finished_dont_incr_completed_twice_if_no_reset(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
finished('link_color', reset=False)
finished('link_color', reset=False)
alternative = Alternative(self.redis, alternative_name, 'link_color')
completion_count = alternative.completed_count
assert completion_count == 1
def test_z_score_when_control_and_alternative_have_perfect_conversion(
self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.save()
control = Alternative(self.redis, 'blue', 'link_color')
control.completed_count = 10
control.participant_count = 10
alternative = Alternative(self.redis, 'red', 'link_color')
alternative.completed_count = 8
alternative.participant_count = 8
assert alternative.z_score is None
def test_can_be_reset(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
alternative.participant_count = 10
alternative.completed_count = 4
alternative.reset()
assert alternative.participant_count == 0
assert alternative.completed_count == 0
def test_z_score_when_control_and_alternative_have_perfect_conversion(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.save()
control = Alternative(self.redis, 'blue', 'link_color')
control.completed_count = 10
control.participant_count = 10
alternative = Alternative(self.redis, 'red', 'link_color')
alternative.completed_count = 8
alternative.participant_count = 8
assert alternative.z_score is None
def test_reset_an_experiment(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
red.participant_count = 5
blue.participant_count = 6
response = self.client.post('/split/link_color/reset')
assert_redirects(response, '/split/')
assert red.participant_count == 0
assert blue.participant_count == 0
def test_reset_an_experiment_if_loaded_with_different_alternatives(self):
experiment = Experiment(self.redis, 'link_color', 'blue', 'red',
'green')
experiment.save()
blue = Alternative(self.redis, 'blue', 'link_color')
blue.participant_count = 5
blue.save()
same_experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'yellow', 'orange')
alternative_names = [a.name for a in same_experiment.alternatives]
assert alternative_names == ['blue', 'yellow', 'orange']
new_blue = Alternative(self.redis, 'blue', 'link_color')
assert new_blue.participant_count == 0
def test_z_score_is_none_when_alternative_has_no_participations(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.save()
alternative = Alternative(self.redis, 'red', 'link_color')
assert alternative.z_score is None
def test_conversion_rate_does_something(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
alternative.participant_count = 10
alternative.completed_count = 4
assert alternative.conversion_rate == 0.4
def test_z_score(self):
Experiment.find_or_create(self.redis, 'Treatment', 'Control',
'Treatment A', 'Treatment B', 'Treatment C')
control = Alternative(self.redis, 'Control', 'Treatment')
control.participant_count = 182
control.completed_count = 35
treatment_a = Alternative(self.redis, 'Treatment A', 'Treatment')
treatment_a.participant_count = 180
treatment_a.completed_count = 45
treatment_b = Alternative(self.redis, 'Treatment B', 'Treatment')
treatment_b.participant_count = 189
treatment_b.completed_count = 28
treatment_c = Alternative(self.redis, 'Treatment C', 'Treatment')
treatment_c.participant_count = 188
treatment_c.completed_count = 61
assert control.z_score is None
assert round(treatment_a.z_score, 2) == 1.33
assert round(treatment_b.z_score, 2) == -1.13
assert round(treatment_c.z_score, 2) == 2.94
def test_saves_to_redis(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
alternative.save()
assert 'basket_text:Basket' in self.redis
def test_belong_to_an_experiment(self):
experiment = Experiment(self.redis, 'basket_text', 'Basket', 'Cart')
experiment.save()
alternative = Alternative(self.redis, 'Basket', 'basket_text')
assert alternative.experiment.name == experiment.name
def test_saves_to_redis(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
alternative.save()
assert 'basket_text:Basket' in self.redis
def test_conversion_rate_does_something(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
alternative.participant_count = 10
alternative.completed_count = 4
assert alternative.conversion_rate == 0.4
def test_has_name(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
assert alternative.name == 'Basket'
def test_has_default_participation_count_of_0(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
assert alternative.participant_count == 0
def test_conversion_rate_is_0_if_there_are_no_conversions(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
assert alternative.completed_count == 0
assert alternative.conversion_rate == 0
def test_has_default_completed_count_of_0(self):
alternative = Alternative(self.redis, 'Basket', 'basket_text')
assert alternative.completed_count == 0
def test_z_score(self):
Experiment.find_or_create(self.redis, 'Treatment',
'Control', 'Treatment A', 'Treatment B', 'Treatment C')
control = Alternative(self.redis, 'Control', 'Treatment')
control.participant_count = 182
control.completed_count = 35
treatment_a = Alternative(self.redis, 'Treatment A', 'Treatment')
treatment_a.participant_count = 180
treatment_a.completed_count = 45
treatment_b = Alternative(self.redis, 'Treatment B', 'Treatment')
treatment_b.participant_count = 189
treatment_b.completed_count = 28
treatment_c = Alternative(self.redis, 'Treatment C', 'Treatment')
treatment_c.participant_count = 188
treatment_c.completed_count = 61
assert control.z_score is None
assert round(treatment_a.z_score, 2) == 1.33
assert round(treatment_b.z_score, 2) == -1.13
assert round(treatment_c.z_score, 2) == 2.94
