def test_bucket__experiment_in_group(self): """ Test that for provided bucket values correct variation ID is returned. """ # In group, matching experiment and variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 4242]) as mock_generate_bucket_value: self.assertEqual(entities.Variation('28902', 'group_exp_1_variation'), self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user')) self.assertEqual([mock.call('test_user19228'), mock.call('test_user32222')], mock_generate_bucket_value.call_args_list) # In group, no matching experiment with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 9500]) as mock_generate_bucket_value: self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user')) self.assertEqual([mock.call('test_user19228'), mock.call('test_user32222')], mock_generate_bucket_value.call_args_list) # In group, experiment does not match with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 4242]) as mock_generate_bucket_value: self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_2'), 'test_user')) mock_generate_bucket_value.assert_called_once_with('test_user19228') # In group no matching variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 424242]) as mock_generate_bucket_value: self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user')) self.assertEqual([mock.call('test_user19228'), mock.call('test_user32222')], mock_generate_bucket_value.call_args_list)
def test_bucket(self): """ Test that for provided bucket value correct variation ID is returned. """ # Variation 1 with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=42) as mock_generate_bucket_value: self.assertEqual( entities.Variation('111128', 'control'), self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_generate_bucket_value.assert_called_once_with('test_user111127') # Empty entity ID with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=4242) as mock_generate_bucket_value: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_generate_bucket_value.assert_called_once_with('test_user111127') # Variation 2 with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=5042) as mock_generate_bucket_value: self.assertEqual( entities.Variation('111129', 'variation'), self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_generate_bucket_value.assert_called_once_with('test_user111127') # No matching variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=424242) as mock_generate_bucket_value: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_generate_bucket_value.assert_called_once_with('test_user111127')
def test_get_forced_variation__user_in_forced_variation(self): """ Test that expected variation is returned if user is forced in a variation. """ experiment = self.project_config.get_experiment_from_key('test_experiment') with mock.patch.object(self.decision_service, 'logger') as mock_decision_logging: self.assertEqual(entities.Variation('111128', 'control'), self.decision_service.get_forced_variation(experiment, 'user_1')) mock_decision_logging.info.assert_called_once_with( 'User "user_1" is forced in variation "control".' )
def test_get_variation__ignore_user_profile_when_specified(self): """ Test that we ignore the user profile service if specified. """ experiment = self.project_config.get_experiment_from_key('test_experiment') with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation', return_value=None) as mock_get_forced_variation, \ mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \ mock.patch('optimizely.bucketer.Bucketer.bucket', return_value=entities.Variation('111129', 'variation')) as mock_bucket, \ mock.patch('optimizely.user_profile.UserProfileService.lookup') as mock_lookup, \ mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save: self.assertEqual(entities.Variation('111129', 'variation'), self.decision_service.get_variation(experiment, 'test_user', None, ignore_user_profile=True)) # Assert that user is bucketed and new decision is NOT stored mock_get_forced_variation.assert_called_once_with(experiment, 'test_user') mock_audience_check.assert_called_once_with(self.project_config, experiment, None) mock_bucket.assert_called_once_with(experiment, 'test_user', 'test_user') self.assertEqual(0, mock_lookup.call_count) self.assertEqual(0, mock_save.call_count)
def test_get_variation__user_profile_lookup_fails(self): """ Test that get_variation acts gracefully when lookup fails. """ experiment = self.project_config.get_experiment_from_key( 'test_experiment') with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation', return_value=None) as mock_get_forced_variation, \ mock.patch('optimizely.decision_service.DecisionService.get_stored_variation') as mock_get_stored_variation, \ mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \ mock.patch('optimizely.bucketer.Bucketer.bucket', return_value=entities.Variation('111129', 'variation')) as mock_bucket, \ mock.patch('optimizely.logger.NoOpLogger.log') as mock_logging, \ mock.patch('optimizely.user_profile.UserProfileService.lookup', side_effect=Exception('major problem')) as mock_lookup, \ mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save: self.assertEqual( entities.Variation('111129', 'variation'), self.decision_service.get_variation(experiment, 'test_user', None)) # Assert that user is bucketed and new decision is stored mock_get_forced_variation.assert_called_once_with( experiment, 'test_user') mock_lookup.assert_called_once_with('test_user') # Stored decision is not consulted as lookup failed self.assertEqual(0, mock_get_stored_variation.call_count) mock_audience_check.assert_called_once_with(self.project_config, experiment, None) mock_logging.assert_any_call( enums.LogLevels.ERROR, 'Unable to retrieve user profile for user "test_user" as lookup failed. Error: major problem' ) mock_bucket.assert_called_once_with(experiment, 'test_user') mock_save.assert_called_once_with({ 'user_id': 'test_user', 'experiment_bucket_map': { '111127': { 'variation_id': '111129' } } })
def test_get_stored_variation__stored_decision_available(self): """ Test that stored decision is retrieved as expected. """ experiment = self.project_config.get_experiment_from_key('test_experiment') profile = user_profile.UserProfile('test_user', experiment_bucket_map={'111127': {'variation_id': '111128'}}) with mock.patch.object(self.decision_service, 'logger') as mock_decision_logging: self.assertEqual(entities.Variation('111128', 'control'), self.decision_service.get_stored_variation(experiment, profile)) mock_decision_logging.info.assert_called_once_with( 'Found a stored decision. User "test_user" is in variation "control" of experiment "test_experiment".' )
def test_get_variation__user_forced_in_variation(self): """ Test that get_variation returns forced variation if user is forced in a variation. """ experiment = self.project_config.get_experiment_from_key('test_experiment') with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation', return_value=entities.Variation('111128', 'control')) as mock_get_forced_variation, \ mock.patch('optimizely.decision_service.DecisionService.get_stored_variation') as mock_get_stored_variation, \ mock.patch('optimizely.helpers.audience.is_user_in_experiment') as mock_audience_check, \ mock.patch('optimizely.bucketer.Bucketer.bucket') as mock_bucket, \ mock.patch('optimizely.user_profile.UserProfileService.lookup') as mock_lookup, \ mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save: self.assertEqual(entities.Variation('111128', 'control'), self.decision_service.get_variation(experiment, 'test_user', None)) # Assert that forced variation is returned and stored decision or bucketing service are not involved mock_get_forced_variation.assert_called_once_with(experiment, 'test_user') self.assertEqual(0, mock_get_stored_variation.call_count) self.assertEqual(0, mock_audience_check.call_count) self.assertEqual(0, mock_bucket.call_count) self.assertEqual(0, mock_lookup.call_count) self.assertEqual(0, mock_save.call_count)
def test_get_variation__user_profile_in_invalid_format(self): """ Test that get_variation handles invalid user profile gracefully. """ experiment = self.project_config.get_experiment_from_key( 'test_experiment') with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation', return_value=None) as mock_get_forced_variation, \ mock.patch('optimizely.decision_service.DecisionService.get_stored_variation') as mock_get_stored_variation, \ mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \ mock.patch('optimizely.bucketer.Bucketer.bucket', return_value=entities.Variation('111129', 'variation')) as mock_bucket, \ mock.patch.object(self.decision_service, 'logger') as mock_decision_logging, \ mock.patch('optimizely.user_profile.UserProfileService.lookup', return_value='invalid_profile') as mock_lookup, \ mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save: self.assertEqual( entities.Variation('111129', 'variation'), self.decision_service.get_variation(experiment, 'test_user', None)) # Assert that user is bucketed and new decision is stored mock_get_forced_variation.assert_called_once_with( experiment, 'test_user') mock_lookup.assert_called_once_with('test_user') # Stored decision is not consulted as user profile is invalid self.assertEqual(0, mock_get_stored_variation.call_count) mock_audience_check.assert_called_once_with(self.project_config, experiment, None) mock_decision_logging.warning.assert_called_once_with( 'User profile has invalid format.') mock_bucket.assert_called_once_with(experiment, 'test_user', 'test_user') mock_save.assert_called_once_with({ 'user_id': 'test_user', 'experiment_bucket_map': { '111127': { 'variation_id': '111129' } } })
def test_get_variation__user_bucketed_for_new_experiment__user_profile_service_available( self): """ Test that get_variation buckets and returns variation if no forced variation or decision available. Also, stores decision if user profile service is available. """ experiment = self.project_config.get_experiment_from_key( 'test_experiment') with mock.patch('optimizely.decision_service.DecisionService.get_forced_variation', return_value=None) as mock_get_forced_variation, \ mock.patch('optimizely.decision_service.DecisionService.get_stored_variation', return_value=None) as mock_get_stored_variation, \ mock.patch('optimizely.helpers.audience.is_user_in_experiment', return_value=True) as mock_audience_check, \ mock.patch('optimizely.bucketer.Bucketer.bucket', return_value=entities.Variation('111129', 'variation')) as mock_bucket, \ mock.patch('optimizely.user_profile.UserProfileService.lookup', return_value={'user_id': 'test_user', 'experiment_bucket_map': {}}) as mock_lookup, \ mock.patch('optimizely.user_profile.UserProfileService.save') as mock_save: self.assertEqual( entities.Variation('111129', 'variation'), self.decision_service.get_variation(experiment, 'test_user', None)) # Assert that user is bucketed and new decision is stored mock_get_forced_variation.assert_called_once_with( experiment, 'test_user') mock_lookup.assert_called_once_with('test_user') self.assertEqual(1, mock_get_stored_variation.call_count) mock_audience_check.assert_called_once_with(self.project_config, experiment, None) mock_bucket.assert_called_once_with(experiment, 'test_user', 'test_user') mock_save.assert_called_once_with({ 'user_id': 'test_user', 'experiment_bucket_map': { '111127': { 'variation_id': '111129' } } })
def test_get_forced_variation__user_in_forced_variation(self): """ Test that expected variation is returned if user is forced in a variation. """ experiment = self.project_config.get_experiment_from_key( 'test_experiment') with mock.patch('optimizely.logger.NoOpLogger.log') as mock_logging: self.assertEqual( entities.Variation('111128', 'control'), self.decision_service.get_forced_variation( experiment, 'user_1')) mock_logging.assert_called_with( enums.LogLevels.INFO, 'User "user_1" is forced in variation "control".')
def test_bucket__experiment_in_group(self): """ Test that for provided bucket values correct variation ID is returned. """ # In group, matching experiment and variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 4242]),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertEqual( entities.Variation('28902', 'group_exp_1_variation'), self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key('group_exp_1'), 'test_user', 'test_user')) mock_config_logging.debug.assert_has_calls([ mock.call( 'Assigned bucket 42 to user with bucketing ID "test_user".'), mock.call( 'Assigned bucket 4242 to user with bucketing ID "test_user".') ]) mock_config_logging.info.assert_has_calls([ mock.call( 'User "test_user" is in experiment group_exp_1 of group 19228.' ), mock.call( 'User "test_user" is in variation "group_exp_1_variation" of experiment group_exp_1.' ) ]) # In group, but in no experiment with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[8400, 9500]),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key('group_exp_1'), 'test_user', 'test_user')) mock_config_logging.debug.assert_called_once_with( 'Assigned bucket 8400 to user with bucketing ID "test_user".') mock_config_logging.info.assert_called_once_with( 'User "test_user" is in no experiment.') # In group, no matching experiment with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 9500]),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key('group_exp_1'), 'test_user', 'test_user')) mock_config_logging.debug.assert_has_calls([ mock.call( 'Assigned bucket 42 to user with bucketing ID "test_user".'), mock.call( 'Assigned bucket 9500 to user with bucketing ID "test_user".') ]) mock_config_logging.info.assert_has_calls([ mock.call( 'User "test_user" is in experiment group_exp_1 of group 19228.' ), mock.call('User "test_user" is in no variation.') ]) # In group, experiment does not match with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 4242]),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key('group_exp_2'), 'test_user', 'test_user')) mock_config_logging.debug.assert_called_once_with( 'Assigned bucket 42 to user with bucketing ID "test_user".') mock_config_logging.info.assert_called_once_with( 'User "test_user" is not in experiment "group_exp_2" of group 19228.' ) # In group no matching variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 424242]),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key('group_exp_1'), 'test_user', 'test_user')) mock_config_logging.debug.assert_has_calls([ mock.call( 'Assigned bucket 42 to user with bucketing ID "test_user".'), mock.call( 'Assigned bucket 424242 to user with bucketing ID "test_user".' ) ]) mock_config_logging.info.assert_has_calls([ mock.call( 'User "test_user" is in experiment group_exp_1 of group 19228.' ), mock.call('User "test_user" is in no variation.') ])
def test_bucket(self): """ Test that expected log messages are logged during bucketing. """ # Variation 1 with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=42),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertEqual( entities.Variation('111128', 'control'), self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_config_logging.debug.assert_called_once_with( 'Assigned bucket 42 to user with bucketing ID "test_user".') mock_config_logging.info.assert_called_once_with( 'User "test_user" is in variation "control" of experiment test_experiment.' ) # Empty entity ID with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=4242), \ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_config_logging.debug.assert_called_once_with( 'Assigned bucket 4242 to user with bucketing ID "test_user".') mock_config_logging.info.assert_called_once_with( 'User "test_user" is in no variation.') # Variation 2 with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=5042),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertEqual( entities.Variation('111129', 'variation'), self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_config_logging.debug.assert_called_once_with( 'Assigned bucket 5042 to user with bucketing ID "test_user".') mock_config_logging.info.assert_called_once_with( 'User "test_user" is in variation "variation" of experiment test_experiment.' ) # No matching variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', return_value=424242),\ mock.patch.object(self.project_config, 'logger') as mock_config_logging: self.assertIsNone( self.bucketer.bucket( self.project_config, self.project_config.get_experiment_from_key( 'test_experiment'), 'test_user', 'test_user')) mock_config_logging.debug.assert_called_once_with( 'Assigned bucket 424242 to user with bucketing ID "test_user".') mock_config_logging.info.assert_called_once_with( 'User "test_user" is in no variation.')
def test_bucket__experiment_in_group(self): """ Test that for provided bucket values correct variation ID is returned. """ # In group, matching experiment and variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 4242]),\ mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging: self.assertEqual(entities.Variation('28902', 'group_exp_1_variation'), self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user')) self.assertEqual(4, mock_logging.call_count) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'), mock_logging.call_args_list[0]) self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in experiment group_exp_1 of group 19228.'), mock_logging.call_args_list[1]) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 4242 to user "test_user".'), mock_logging.call_args_list[2]) self.assertEqual( mock.call(enums.LogLevels.INFO, 'User "test_user" is in variation "group_exp_1_variation" of experiment group_exp_1.'), mock_logging.call_args_list[3] ) # In group, but in no experiment with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[8400, 9500]),\ mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging: self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user')) self.assertEqual(2, mock_logging.call_count) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 8400 to user "test_user".'), mock_logging.call_args_list[0]) self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in no experiment.'), mock_logging.call_args_list[1]) # In group, no matching experiment with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 9500]),\ mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging: self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user')) self.assertEqual(4, mock_logging.call_count) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'), mock_logging.call_args_list[0]) self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in experiment group_exp_1 of group 19228.'), mock_logging.call_args_list[1]) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 9500 to user "test_user".'), mock_logging.call_args_list[2]) self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in no variation.'), mock_logging.call_args_list[3]) # In group, experiment does not match with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 4242]),\ mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging: self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_2'), 'test_user')) self.assertEqual(2, mock_logging.call_count) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'), mock_logging.call_args_list[0]) self.assertEqual( mock.call(enums.LogLevels.INFO, 'User "test_user" is not in experiment "group_exp_2" of group 19228.'), mock_logging.call_args_list[1] ) # In group no matching variation with mock.patch('optimizely.bucketer.Bucketer._generate_bucket_value', side_effect=[42, 424242]),\ mock.patch('optimizely.logger.SimpleLogger.log') as mock_logging: self.assertIsNone(self.bucketer.bucket(self.project_config.get_experiment_from_key('group_exp_1'), 'test_user')) self.assertEqual(4, mock_logging.call_count) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 42 to user "test_user".'), mock_logging.call_args_list[0]) self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in experiment group_exp_1 of group 19228.'), mock_logging.call_args_list[1]) self.assertEqual(mock.call(enums.LogLevels.DEBUG, 'Assigned bucket 424242 to user "test_user".'), mock_logging.call_args_list[2]) self.assertEqual(mock.call(enums.LogLevels.INFO, 'User "test_user" is in no variation.'), mock_logging.call_args_list[3])