def test_bucket__experiment_in_group(self):
""" Test that for provided bucket values correct variation ID is returned. """
# In group, matching experiment and variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 4242]) as mock_generate_bucket_value:
self.assertEqual(entities.Variation('28902', 'group_exp_1_variation'),
self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user'))
self.assertEqual([mock.call('test_user19228'), mock.call('test_user32222')],
mock_generate_bucket_value.call_args_list)
# In group, no matching experiment
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 9500]) as mock_generate_bucket_value:
self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'),
'test_user'))
self.assertEqual([mock.call('test_user19228'), mock.call('test_user32222')],
mock_generate_bucket_value.call_args_list)
# In group, experiment does not match
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 4242]) as mock_generate_bucket_value:
self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_2'),
'test_user'))
mock_generate_bucket_value.assert_called_once_with('test_user19228')
# In group no matching variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 424242]) as mock_generate_bucket_value:
self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'),
'test_user'))
self.assertEqual([mock.call('test_user19228'), mock.call('test_user32222')],
mock_generate_bucket_value.call_args_list)
def test_bucket(self):
""" Test that for provided bucket value correct variation ID is returned. """
# Variation 1
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
return_value=42) as mock_generate_bucket_value:
self.assertEqual(
entities.Variation('111128', 'control'),
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_generate_bucket_value.assert_called_once_with('test_user111127')
# Empty entity ID
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
return_value=4242) as mock_generate_bucket_value:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_generate_bucket_value.assert_called_once_with('test_user111127')
# Variation 2
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
return_value=5042) as mock_generate_bucket_value:
self.assertEqual(
entities.Variation('111129', 'variation'),
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_generate_bucket_value.assert_called_once_with('test_user111127')
# No matching variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
return_value=424242) as mock_generate_bucket_value:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_generate_bucket_value.assert_called_once_with('test_user111127')
def test_get_forced_variation__user_in_forced_variation(self):
""" Test that expected variation is returned if user is forced in a variation. """
experiment = self.project_config.get_experiment_from_key('test_experiment')
with mock.patch.object(self.decision_service, 'logger') as mock_decision_logging:
self.assertEqual(entities.Variation('111128', 'control'),
self.decision_service.get_forced_variation(experiment, 'user_1'))
mock_decision_logging.info.assert_called_once_with(
'User "user_1" is forced in variation "control".'
)
def test_get_variation__ignore_user_profile_when_specified(self):
""" Test that we ignore the user profile service if specified. """
experiment = self.project_config.get_experiment_from_key('test_experiment')
with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation',
return_value=None) as mock_get_forced_variation, \
mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \
mock.patch('optimizely.bucketer.Bucketer.bucket',
return_value=entities.Variation('111129', 'variation')) as mock_bucket, \
mock.patch('optimizely.user_profile.UserProfileService.lookup') as mock_lookup, \
mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save:
self.assertEqual(entities.Variation('111129', 'variation'),
self.decision_service.get_variation(experiment, 'test_user', None, ignore_user_profile=True))
# Assert that user is bucketed and new decision is NOT stored
mock_get_forced_variation.assert_called_once_with(experiment, 'test_user')
mock_audience_check.assert_called_once_with(self.project_config, experiment, None)
mock_bucket.assert_called_once_with(experiment, 'test_user', 'test_user')
self.assertEqual(0, mock_lookup.call_count)
self.assertEqual(0, mock_save.call_count)
def test_get_variation__user_profile_lookup_fails(self):
""" Test that get_variation acts gracefully when lookup fails. """
experiment = self.project_config.get_experiment_from_key(
'test_experiment')
with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation',
return_value=None) as mock_get_forced_variation, \
mock.patch('optimizely.decision_service.DecisionService.get_stored_variation') as mock_get_stored_variation, \
mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \
mock.patch('optimizely.bucketer.Bucketer.bucket',
return_value=entities.Variation('111129', 'variation')) as mock_bucket, \
mock.patch('optimizely.logger.NoOpLogger.log') as mock_logging, \
mock.patch('optimizely.user_profile.UserProfileService.lookup',
side_effect=Exception('major problem')) as mock_lookup, \
mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save:
self.assertEqual(
entities.Variation('111129', 'variation'),
self.decision_service.get_variation(experiment, 'test_user',
None))
# Assert that user is bucketed and new decision is stored
mock_get_forced_variation.assert_called_once_with(
experiment, 'test_user')
mock_lookup.assert_called_once_with('test_user')
# Stored decision is not consulted as lookup failed
self.assertEqual(0, mock_get_stored_variation.call_count)
mock_audience_check.assert_called_once_with(self.project_config,
experiment, None)
mock_logging.assert_any_call(
enums.LogLevels.ERROR,
'Unable to retrieve user profile for user "test_user" as lookup failed. Error: major problem'
)
mock_bucket.assert_called_once_with(experiment, 'test_user')
mock_save.assert_called_once_with({
'user_id': 'test_user',
'experiment_bucket_map': {
'111127': {
'variation_id': '111129'
}
}
})
def test_get_stored_variation__stored_decision_available(self):
""" Test that stored decision is retrieved as expected. """
experiment = self.project_config.get_experiment_from_key('test_experiment')
profile = user_profile.UserProfile('test_user', experiment_bucket_map={'111127': {'variation_id': '111128'}})
with mock.patch.object(self.decision_service, 'logger') as mock_decision_logging:
self.assertEqual(entities.Variation('111128', 'control'),
self.decision_service.get_stored_variation(experiment, profile))
mock_decision_logging.info.assert_called_once_with(
'Found a stored decision. User "test_user" is in variation "control" of experiment "test_experiment".'
)
def test_get_variation__user_forced_in_variation(self):
""" Test that get_variation returns forced variation if user is forced in a variation. """
experiment = self.project_config.get_experiment_from_key('test_experiment')
with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation',
return_value=entities.Variation('111128', 'control')) as mock_get_forced_variation, \
mock.patch('optimizely.decision_service.DecisionService.get_stored_variation') as mock_get_stored_variation, \
mock.patch('optimizely.helpers.audience.is_user_in_experiment') as mock_audience_check, \
mock.patch('optimizely.bucketer.Bucketer.bucket') as mock_bucket, \
mock.patch('optimizely.user_profile.UserProfileService.lookup') as mock_lookup, \
mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save:
self.assertEqual(entities.Variation('111128', 'control'),
self.decision_service.get_variation(experiment, 'test_user', None))
# Assert that forced variation is returned and stored decision or bucketing service are not involved
mock_get_forced_variation.assert_called_once_with(experiment, 'test_user')
self.assertEqual(0, mock_get_stored_variation.call_count)
self.assertEqual(0, mock_audience_check.call_count)
self.assertEqual(0, mock_bucket.call_count)
self.assertEqual(0, mock_lookup.call_count)
self.assertEqual(0, mock_save.call_count)
def test_get_variation__user_profile_in_invalid_format(self):
""" Test that get_variation handles invalid user profile gracefully. """
experiment = self.project_config.get_experiment_from_key(
'test_experiment')
with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation',
return_value=None) as mock_get_forced_variation, \
mock.patch('optimizely.decision_service.DecisionService.get_stored_variation') as mock_get_stored_variation, \
mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \
mock.patch('optimizely.bucketer.Bucketer.bucket',
return_value=entities.Variation('111129', 'variation')) as mock_bucket, \
mock.patch.object(self.decision_service, 'logger') as mock_decision_logging, \
mock.patch('optimizely.user_profile.UserProfileService.lookup',
return_value='invalid_profile') as mock_lookup, \
mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save:
self.assertEqual(
entities.Variation('111129', 'variation'),
self.decision_service.get_variation(experiment, 'test_user',
None))
# Assert that user is bucketed and new decision is stored
mock_get_forced_variation.assert_called_once_with(
experiment, 'test_user')
mock_lookup.assert_called_once_with('test_user')
# Stored decision is not consulted as user profile is invalid
self.assertEqual(0, mock_get_stored_variation.call_count)
mock_audience_check.assert_called_once_with(self.project_config,
experiment, None)
mock_decision_logging.warning.assert_called_once_with(
'User profile has invalid format.')
mock_bucket.assert_called_once_with(experiment, 'test_user',
'test_user')
mock_save.assert_called_once_with({
'user_id': 'test_user',
'experiment_bucket_map': {
'111127': {
'variation_id': '111129'
}
}
})
def test_get_variation__user_bucketed_for_new_experiment__user_profile_service_available(
self):
""" Test that get_variation buckets and returns variation if no forced variation or decision available.
Also, stores decision if user profile service is available. """
experiment = self.project_config.get_experiment_from_key(
'test_experiment')
with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation',
return_value=None) as mock_get_forced_variation, \
mock.patch('optimizely.decision_service.DecisionService.get_stored_variation',
return_value=None) as mock_get_stored_variation, \
mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \
mock.patch('optimizely.bucketer.Bucketer.bucket',
return_value=entities.Variation('111129', 'variation')) as mock_bucket, \
mock.patch('optimizely.user_profile.UserProfileService.lookup',
return_value={'user_id': 'test_user', 'experiment_bucket_map': {}}) as mock_lookup, \
mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save:
self.assertEqual(
entities.Variation('111129', 'variation'),
self.decision_service.get_variation(experiment, 'test_user',
None))
# Assert that user is bucketed and new decision is stored
mock_get_forced_variation.assert_called_once_with(
experiment, 'test_user')
mock_lookup.assert_called_once_with('test_user')
self.assertEqual(1, mock_get_stored_variation.call_count)
mock_audience_check.assert_called_once_with(self.project_config,
experiment, None)
mock_bucket.assert_called_once_with(experiment, 'test_user',
'test_user')
mock_save.assert_called_once_with({
'user_id': 'test_user',
'experiment_bucket_map': {
'111127': {
'variation_id': '111129'
}
}
})
def test_get_forced_variation__user_in_forced_variation(self):
""" Test that expected variation is returned if user is forced in a variation. """
experiment = self.project_config.get_experiment_from_key(
'test_experiment')
with mock.patch('optimizely.logger.NoOpLogger.log') as mock_logging:
self.assertEqual(
entities.Variation('111128', 'control'),
self.decision_service.get_forced_variation(
experiment, 'user_1'))
mock_logging.assert_called_with(
enums.LogLevels.INFO,
'User "user_1" is forced in variation "control".')
def test_bucket__experiment_in_group(self):
""" Test that for provided bucket values correct variation ID is returned. """
# In group, matching experiment and variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 4242]),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertEqual(
entities.Variation('28902', 'group_exp_1_variation'),
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key('group_exp_1'),
'test_user', 'test_user'))
mock_config_logging.debug.assert_has_calls([
mock.call(
'Assigned bucket 42 to user with bucketing ID "test_user".'),
mock.call(
'Assigned bucket 4242 to user with bucketing ID "test_user".')
])
mock_config_logging.info.assert_has_calls([
mock.call(
'User "test_user" is in experiment group_exp_1 of group 19228.'
),
mock.call(
'User "test_user" is in variation "group_exp_1_variation" of experiment group_exp_1.'
)
])
# In group, but in no experiment
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[8400, 9500]),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key('group_exp_1'),
'test_user', 'test_user'))
mock_config_logging.debug.assert_called_once_with(
'Assigned bucket 8400 to user with bucketing ID "test_user".')
mock_config_logging.info.assert_called_once_with(
'User "test_user" is in no experiment.')
# In group, no matching experiment
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 9500]),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key('group_exp_1'),
'test_user', 'test_user'))
mock_config_logging.debug.assert_has_calls([
mock.call(
'Assigned bucket 42 to user with bucketing ID "test_user".'),
mock.call(
'Assigned bucket 9500 to user with bucketing ID "test_user".')
])
mock_config_logging.info.assert_has_calls([
mock.call(
'User "test_user" is in experiment group_exp_1 of group 19228.'
),
mock.call('User "test_user" is in no variation.')
])
# In group, experiment does not match
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 4242]),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key('group_exp_2'),
'test_user', 'test_user'))
mock_config_logging.debug.assert_called_once_with(
'Assigned bucket 42 to user with bucketing ID "test_user".')
mock_config_logging.info.assert_called_once_with(
'User "test_user" is not in experiment "group_exp_2" of group 19228.'
)
# In group no matching variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 424242]),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key('group_exp_1'),
'test_user', 'test_user'))
mock_config_logging.debug.assert_has_calls([
mock.call(
'Assigned bucket 42 to user with bucketing ID "test_user".'),
mock.call(
'Assigned bucket 424242 to user with bucketing ID "test_user".'
)
])
mock_config_logging.info.assert_has_calls([
mock.call(
'User "test_user" is in experiment group_exp_1 of group 19228.'
),
mock.call('User "test_user" is in no variation.')
])
def test_bucket(self):
""" Test that expected log messages are logged during bucketing. """
# Variation 1
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=42),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertEqual(
entities.Variation('111128', 'control'),
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_config_logging.debug.assert_called_once_with(
'Assigned bucket 42 to user with bucketing ID "test_user".')
mock_config_logging.info.assert_called_once_with(
'User "test_user" is in variation "control" of experiment test_experiment.'
)
# Empty entity ID
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=4242), \
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_config_logging.debug.assert_called_once_with(
'Assigned bucket 4242 to user with bucketing ID "test_user".')
mock_config_logging.info.assert_called_once_with(
'User "test_user" is in no variation.')
# Variation 2
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=5042),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertEqual(
entities.Variation('111129', 'variation'),
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_config_logging.debug.assert_called_once_with(
'Assigned bucket 5042 to user with bucketing ID "test_user".')
mock_config_logging.info.assert_called_once_with(
'User "test_user" is in variation "variation" of experiment test_experiment.'
)
# No matching variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=424242),\
mock.patch.object(self.project_config, 'logger') as mock_config_logging:
self.assertIsNone(
self.bucketer.bucket(
self.project_config,
self.project_config.get_experiment_from_key(
'test_experiment'), 'test_user', 'test_user'))
mock_config_logging.debug.assert_called_once_with(
'Assigned bucket 424242 to user with bucketing ID "test_user".')
mock_config_logging.info.assert_called_once_with(
'User "test_user" is in no variation.')
def test_bucket__experiment_in_group(self):
""" Test that for provided bucket values correct variation ID is returned. """
# In group, matching experiment and variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 4242]),\
mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging:
self.assertEqual(entities.Variation('28902', 'group_exp_1_variation'),
self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user'))
self.assertEqual(4, mock_logging.call_count)
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'),
mock_logging.call_args_list[0])
self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in experiment group_exp_1 of group 19228.'),
mock_logging.call_args_list[1])
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 4242 to user "test_user".'),
mock_logging.call_args_list[2])
self.assertEqual(
mock.call(enums.LogLevels.INFO,
'User "test_user" is in variation "group_exp_1_variation" of experiment group_exp_1.'),
mock_logging.call_args_list[3]
)
# In group, but in no experiment
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[8400, 9500]),\
mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging:
self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user'))
self.assertEqual(2, mock_logging.call_count)
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 8400 to user "test_user".'),
mock_logging.call_args_list[0])
self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in no experiment.'),
mock_logging.call_args_list[1])
# In group, no matching experiment
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 9500]),\
mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging:
self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user'))
self.assertEqual(4, mock_logging.call_count)
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'),
mock_logging.call_args_list[0])
self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in experiment group_exp_1 of group 19228.'),
mock_logging.call_args_list[1])
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 9500 to user "test_user".'),
mock_logging.call_args_list[2])
self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in no variation.'),
mock_logging.call_args_list[3])
# In group, experiment does not match
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 4242]),\
mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging:
self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_2'), 'test_user'))
self.assertEqual(2, mock_logging.call_count)
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'),
mock_logging.call_args_list[0])
self.assertEqual(
mock.call(enums.LogLevels.INFO, 'User "test_user" is not in experiment "group_exp_2" of group 19228.'),
mock_logging.call_args_list[1]
)
# In group no matching variation
with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value',
side_effect=[42, 424242]),\
mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging:
self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user'))
self.assertEqual(4, mock_logging.call_count)
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'),
mock_logging.call_args_list[0])
self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in experiment group_exp_1 of group 19228.'),
mock_logging.call_args_list[1])
self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 424242 to user "test_user".'),
mock_logging.call_args_list[2])
self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in no variation.'),
mock_logging.call_args_list[3])
