def test_can_participate_in_many_experiments_with_allow_multiple_experiments(self):
self.app.config['SPLIT_ALLOW_MULTIPLE_EXPERIMENTS'] = True
link_color = ab_test('link_color', 'blue', 'red')
button_size = ab_test('button_size', 'small', 'big')
assert _get_session()['button_size'] == button_size
button_size_alt = Alternative(self.redis, button_size, 'button_size')
assert button_size_alt.participant_count == 1
def test_ab_test_raises_an_exception_without_db_failover(self):
self.app.config['SPLIT_DB_FAILOVER'] = False
(flexmock(Redis)
.should_receive('execute_command')
.and_raise(ConnectionError))
with raises(ConnectionError):
ab_test('link_color', 'blue', 'red')
def test_ab_test_only_lets_user_participate_in_one_experiment(self):
ab_test('link_color', 'blue', 'red')
ab_test('button_size', 'small', 'big')
assert _get_session()['button_size'] == 'small'
big = Alternative(self.redis, 'big', 'button_size')
assert big.participant_count == 0
small = Alternative(self.redis, 'small', 'button_size')
assert small.participant_count == 0
def test_ab_test_always_uses_first_alternative_with_db_failover(self):
self.app.config['SPLIT_DB_FAILOVER'] = True
(flexmock(Redis)
.should_receive('execute_command')
.and_raise(ConnectionError))
assert ab_test('link_color', 'blue', 'red') == 'blue'
assert ab_test('link_color', ('blue', 0.01), ('red', 0.2)) == 'blue'
assert ab_test('link_color', ('blue', 0.8), ('red', 20)) == 'blue'
def test_loads_the_experiment_even_if_the_version_is_not_0(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.reset()
assert experiment.version == 1
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': alternative_name}
return_alternative_name = ab_test('link_color', 'blue', 'red')
assert return_alternative_name == alternative_name
def test_finished_dont_incr_completed_twice_if_ver_gt_0_and_no_reset(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.increment_version()
alternative_name = ab_test('link_color', 'blue', 'red')
finished('link_color', reset=False)
alternative_name = ab_test('link_color', 'blue', 'red')
finished('link_color', reset=False)
alternative = Alternative(self.redis, alternative_name, 'link_color')
completion_count = alternative.completed_count
assert completion_count == 1
def test_cleans_up_old_versions_of_experiments_from_the_session(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': new_alternative_name}
def test_saves_the_version_of_the_experiment_to_the_session(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.reset()
assert experiment.version == 1
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color:1': alternative_name}
def test_ab_test_increments_participation_counter_for_new_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
red = Alternative(self.redis, 'red', 'link_color')
blue = Alternative(self.redis, 'blue', 'link_color')
previous_red_count = red.participant_count
previous_blue_count = blue.participant_count
ab_test('link_color', 'blue', 'red')
new_red_count = red.participant_count
new_blue_count = blue.participant_count
assert (new_red_count + new_blue_count ==
previous_red_count + previous_blue_count + 1)
def test_finished_clears_out_users_participation_from_their_session(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color": alternative_name}
finished('link_color')
assert session['split'] == {}
def test_finished_increments_completed_alternative_counter(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
previous_completion_count = alternative.completed_count
finished('link_color')
new_completion_count = alternative.completed_count
assert new_completion_count == previous_completion_count + 1
def test_resets_users_session_on_an_older_version_of_the_experiment(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 1
experiment.reset()
assert experiment.version == 1
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.participant_count == 0
new_alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split']['link_color:1'] == new_alternative_name
new_alternative = Alternative(
self.redis, new_alternative_name, 'link_color')
assert new_alternative.participant_count == 1
def test_conversions_return_conversion_rates_for_alternatives(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.conversion_rate == 0.0
finished('link_color')
assert alternative.conversion_rate == 1.0
def test_finished_dont_clear_out_the_users_session_if_reset_is_false(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color": alternative_name}
finished('link_color', reset=False)
assert session['split'] == {
"link_color": alternative_name,
}
assert session['split_finished'] == set(['link_color'])
def test_finished_clears_test_session_when_version_is_greater_than_0(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.increment_version()
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color:1": alternative_name}
finished('link_color')
assert session['split'] == {}
def test_finished_clears_test_session_when_version_is_greater_than_0(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.increment_version()
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {"link_color:1": alternative_name}
finished('link_color')
assert session['split'] == {}
def test_finished_does_not_increment_the_completed_count(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
alternative = Alternative(self.redis, alternative_name, 'link_color')
previous_completion_count = alternative.completed_count
finished('link_color')
new_completion_count = alternative.completed_count
assert new_completion_count == previous_completion_count
def test_only_counts_completion_of_users_on_the_current_version(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
experiment.reset()
assert experiment.version == 1
finished('link_color')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.completed_count == 0
def test_only_counts_completion_of_users_on_the_current_version(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert session['split'] == {'link_color': alternative_name}
alternative = Alternative(self.redis, alternative_name, 'link_color')
experiment.reset()
assert experiment.version == 1
finished('link_color')
alternative = Alternative(self.redis, alternative_name, 'link_color')
assert alternative.completed_count == 0
def test_ab_test_does_not_raise_an_exception_with_db_failover(self):
self.app.config['SPLIT_DB_FAILOVER'] = True
(flexmock(Redis).should_receive('execute_command').and_raise(
ConnectionError))
ab_test('link_color', 'blue', 'red')
def test_uses_version_zero_if_no_version_is_present(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert experiment.version == 0
assert session['split'] == {'link_color': alternative_name}
def test_ab_test_return_the_control(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
assert alternative == experiment.control.name
def check_full_page():
return render_template(ab_test('page_checker','page1.html', 'page2.html'))
def test_ab_test_always_returns_the_winner_if_one_is_present(self):
experiment = Experiment.find_or_create(self.redis, 'link_color',
'blue', 'red')
experiment.winner = "orange"
assert ab_test('link_color', 'blue', 'red') == 'orange'
def test_can_serialize_session(self):
ab_test('link_color', 'blue', 'red')
finished('link_color')
self.app.save_session(session, make_response())
def test_ab_test_return_the_control(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
assert alternative == experiment.control.name
def test_ab_test_allows_the_share_of_visitors_see_an_alternative(self):
ab_test('link_color', ('blue', 0.8), ('red', 20))
assert _get_session()['link_color'] in ['red', 'blue']
def test_ab_test_always_returns_the_winner_if_one_is_present(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
experiment.winner = "orange"
assert ab_test('link_color', 'blue', 'red') == 'orange'
def test_ab_test_returns_the_given_alternative_for_an_existing_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
repeat_alternative = ab_test('link_color', 'blue', 'red')
assert alternative == repeat_alternative
def test_uses_version_zero_if_no_version_is_present(self):
experiment = Experiment.find_or_create(
self.redis, 'link_color', 'blue', 'red')
alternative_name = ab_test('link_color', 'blue', 'red')
assert experiment.version == 0
assert session['split'] == {'link_color': alternative_name}
def test_can_serialize_session(self):
ab_test('link_color', 'blue', 'red')
finished('link_color')
self.app.session_interface.save_session(self.app, session, make_response())
def test_ab_test_assigns_random_alternative_to_a_new_user(self):
ab_test('link_color', 'blue', 'red')
assert _get_session()['link_color'] in ['red', 'blue']
def test_ab_test_returns_the_given_alternative_for_an_existing_user(self):
Experiment.find_or_create(self.redis, 'link_color', 'blue', 'red')
alternative = ab_test('link_color', 'blue', 'red')
repeat_alternative = ab_test('link_color', 'blue', 'red')
assert alternative == repeat_alternative
def product():
alternative = ab_test('buy_text', 'Buy', 'Free')
return render_template('product.html',
alternative=alternative
)
def test_ab_test_allows_the_share_of_visitors_see_an_alternative(self):
ab_test('link_color', ('blue', 0.8), ('red', 20))
assert _get_session()['link_color'] in ['red', 'blue']
def test_ab_test_assigns_random_alternative_to_a_new_user(self):
ab_test('link_color', 'blue', 'red')
assert _get_session()['link_color'] in ['red', 'blue']
