def test_bulk_index_error_handling(self): """Check that 404 and 409 errors are appropriately ignored""" from elasticsearch import helpers mock_engine = mock.Mock() plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' with mock.patch(bulk_name) as mock_bulk: mock_bulk.side_effect = helpers.BulkIndexError( "1 document(s) failed to index", [{ 'delete': { "_id": "1", "error": "Some error", "status": 404, "exception": helpers.TransportError() } }]) indexing_helper.delete_documents([{'_id': '1'}]) self.assertEqual(1, mock_bulk.call_count) with mock.patch(bulk_name) as mock_bulk: mock_bulk.side_effect = helpers.BulkIndexError( "1 document(s) failed to index", [{ 'index': { "_id": "1", "error": "VersionConflict", "status": 409 } }]) indexing_helper.save_documents([{'id': '1'}]) self.assertEqual(1, mock_bulk.call_count)
def test_non_role_separated_save_docs(self): """Test that for a plugin that doesn't specify any protected fields, ids are left alone and there's only one copy of each indexed doc """ mock_engine = mock.Mock() plugin = fake_plugins.NonRoleSeparatedPlugin(es_engine=mock_engine) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' with mock.patch(bulk_name) as mock_bulk: count = len(plugin.get_objects()) fake_versions = range(1, count + 1) indexing_helper.save_documents(plugin.get_objects(), fake_versions) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) self.assertEqual(2, len(actions)) self.assertEqual(set(['non-role-fake1', 'non-role-fake2']), set(action['_id'] for action in actions)) self.assertEqual(set(fake_versions), set(action['_version'] for action in actions)) self.assertEqual(['external'] * 2, [action['_version_type'] for action in actions]) fake1 = actions[0]['_source'] self.assertEqual(['admin', 'user'], sorted(fake1.pop(ROLE_USER_FIELD))) self.assertEqual(fake_plugins.NON_ROLE_SEPARATED_DATA[0], actions[0]['_source'])
def test_region_mapping(self, service_credentials_conf, resource_plugin_conf, mock_bulk): mock_engine = mock.Mock() plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) resource_plugin_conf.include_region_name = True service_credentials_conf.os_region_name = 'test-region' indexing_helper = helper.IndexingHelper(plugin) _, mapping = six.next(plugin.get_full_mapping()) self.assertIn('region_name', mapping['properties']) count = len(plugin.get_objects()) fake_versions = range(1, count + 1) indexing_helper.save_documents(plugin.get_objects(), fake_versions) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) self.assertEqual(['test-region'], actions[0]['_source']['region_name']) # Test without a region resource_plugin_conf.include_region_name = False mock_bulk.reset_mock() _, mapping = six.next(plugin.get_full_mapping()) self.assertNotIn('region_name', mapping['properties']) indexing_helper.save_documents(plugin.get_objects(), fake_versions) actions = list(mock_bulk.call_args_list[0][1]['actions']) self.assertNotIn('region_name', actions[0]['_source'])
def test_child_only_separated(self, mock_role_separated): """Test where the child (and not the parent) is separated. Expect two documents with the same parent """ mock_role_separated.return_value = True mock_engine = mock.Mock() plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) parent_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) plugin.register_parent(parent_plugin) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' child_docs = copy.deepcopy(fake_plugins.CHILD_DATA) with mock.patch(bulk_name) as mock_bulk: indexing_helper.save_documents(child_docs) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) expected_admin_doc = copy.deepcopy(child_docs[0]) expected_admin_doc[ROLE_USER_FIELD] = 'admin' expected_user_doc = copy.deepcopy(child_docs[0]) expected_user_doc[ROLE_USER_FIELD] = 'user' expected_actions = [ {'_op_type': 'index', '_id': 'child1_ADMIN', '_source': expected_admin_doc, '_parent': 'simple1'}, {'_op_type': 'index', '_id': 'child1_USER', '_source': expected_user_doc, '_parent': 'simple1'} ] self.assertEqual(expected_actions, list(actions))
def test_save_parent_only_separated(self, mock_role_separated): """Test where the parent document is role separated but this child is not; it's expected the parent's _USER documents be used for _parent """ mock_role_separated.return_value = False mock_engine = mock.Mock() plugin = fake_plugins.FakeSeparatedChildPlugin(es_engine=mock_engine) parent_plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine) plugin.register_parent(parent_plugin) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' child_docs = copy.deepcopy(fake_plugins.CHILD_DATA) with mock.patch(bulk_name) as mock_bulk: indexing_helper.save_documents(child_docs) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) expected_doc = copy.deepcopy(child_docs[0]) expected_doc[ROLE_USER_FIELD] = ['user', 'admin'] expected_actions = [ {'_op_type': 'index', '_id': 'child1', '_source': expected_doc, '_parent': 'simple1_USER'}, ] self.assertEqual(expected_actions, list(actions))
def test_save_child_parent_both_separated(self): mock_engine = mock.Mock() plugin = fake_plugins.FakeSeparatedChildPlugin(es_engine=mock_engine) parent_plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine) plugin.register_parent(parent_plugin) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' child_docs = copy.deepcopy(fake_plugins.CHILD_DATA) # First run where both child and parent are role-separated (and thus # we'd expect two copies of both parent and child with appropriate # ids linking them) with mock.patch(bulk_name) as mock_bulk: indexing_helper.save_documents(child_docs) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) expected_admin_doc = copy.deepcopy(child_docs[0]) expected_admin_doc[ROLE_USER_FIELD] = 'admin' expected_user_doc = copy.deepcopy(child_docs[0]) expected_user_doc[ROLE_USER_FIELD] = 'user' expected_actions = [ {'_op_type': 'index', '_id': 'child1_ADMIN', '_source': expected_admin_doc, '_parent': 'simple1_ADMIN'}, {'_op_type': 'index', '_id': 'child1_USER', '_source': expected_user_doc, '_parent': 'simple1_USER'} ] self.assertEqual(expected_actions, list(actions))
def test_delete_children_role_separated(self): mock_engine = mock.Mock() plugin = fake_plugins.FakeSeparatedChildPlugin(es_engine=mock_engine) parent_plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine) plugin.register_parent(parent_plugin) indexing_helper = helper.IndexingHelper(plugin) scan_name = 'searchlight.elasticsearch.plugins.helper.helpers.scan' bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' mock_scan_data = [{ '_id': '1_ADMIN', 'fields': { '_parent': 'p1_ADMIN' } }, { '_id': '1_USER', 'fields': { '_parent': 'p1_USER' } }] with mock.patch(scan_name, return_value=mock_scan_data) as mock_scan: with mock.patch(bulk_name) as mock_bulk: indexing_helper.delete_documents_with_parent('p1') parent_type = plugin.parent_plugin_type() base_full_parent_type = '%s#%s' % (parent_type, 'p1') expected_scan_query = { 'fields': ['_parent', '_routing'], 'query': { 'terms': { '_parent': [ base_full_parent_type + '_ADMIN', base_full_parent_type + '_USER' ] } } } mock_scan.assert_called_with(client=plugin.engine, index=plugin.alias_name_listener, doc_type=plugin.document_type, query=expected_scan_query) expected_delete_actions = [{ '_op_type': 'delete', '_id': '1_ADMIN', '_parent': 'p1_ADMIN' }, { '_op_type': 'delete', '_id': '1_USER', '_parent': 'p1_USER' }] mock_bulk.assert_called_with(client=plugin.engine, index=plugin.alias_name_listener, doc_type=plugin.document_type, actions=expected_delete_actions)
def test_non_role_separated_delete(self): """Test that deletion for a role-separated plugin deletes the doc""" mock_engine = mock.Mock() plugin = fake_plugins.NonRoleSeparatedPlugin(es_engine=mock_engine) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' with mock.patch(bulk_name) as mock_bulk: indexing_helper.delete_document({'_id': 'non-role-fake1'}) expected_delete_actions = [{ '_op_type': 'delete', '_id': 'non-role-fake1' }] mock_bulk.assert_called_once_with(client=plugin.engine, index=plugin.alias_name_listener, doc_type=plugin.document_type, actions=expected_delete_actions)
def test_role_separated_save_docs(self): """Test admin only fields are correctly removed from serialization and that alternate _id values are used """ mock_engine = mock.Mock() plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' with mock.patch(bulk_name) as mock_bulk: count = len(plugin.get_objects()) fake_versions = range(1, count + 1) indexing_helper.save_documents(plugin.get_objects(), fake_versions) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) self.assertEqual(4, len(actions)) self.assertEqual( set([ 'role-fake1_ADMIN', 'role-fake2_ADMIN', 'role-fake1_USER', 'role-fake2_USER' ]), set(action['_id'] for action in actions)) self.assertEqual(set(fake_versions), set(action['_version'] for action in actions)) self.assertEqual(['external'] * 4, [action['_version_type'] for action in actions]) # This plugin filters on admin_wildcard_* and admin_specific fake1_admin = list( filter(lambda a: a['_id'] == 'role-fake1_ADMIN', actions))[0]['_source'] self.assertEqual('role-fake1', fake1_admin['id']) self.assertIn('public_field', fake1_admin) self.assertIn('admin_wildcard_this', fake1_admin) self.assertEqual('admin', fake1_admin[ROLE_USER_FIELD]) fake1_user = list( filter(lambda a: a['_id'] == 'role-fake1_USER', actions))[0]['_source'] self.assertEqual('role-fake1', fake1_user['id']) self.assertIn('public_field', fake1_user) self.assertNotIn('admin_wildcard_this', fake1_user) self.assertNotIn('admin_specific', fake1_user) self.assertEqual('user', fake1_user[ROLE_USER_FIELD])
def test_routing_save_docs(self): """Test that for a plugin that specifies routing_id field will end up with "_routing" set while indexing. """ mock_engine = mock.Mock() plugin = fake_plugins.FakeSimpleRoutingPlugin(es_engine=mock_engine) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' with mock.patch(bulk_name) as mock_bulk: count = len(plugin.get_objects()) fake_versions = range(1, count + 1) indexing_helper.save_documents(plugin.get_objects(), fake_versions) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) # '_routing' is added to action only if set in the plugin property # FakeSimpleRoutingPlugin has it defined. self.assertIs(True, '_routing' in actions[0]) self.assertEqual('tenant1', actions[0]['_routing'])
def test_routing_delete(self): """Test that deletion for a routing based plugin deletes docs""" mock_engine = mock.Mock() plugin = fake_plugins.FakeSimpleRoutingPlugin(es_engine=mock_engine) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' with mock.patch(bulk_name) as mock_bulk: indexing_helper.delete_document({ '_id': 'id_for_routing_plugin-fake1', '_routing': 'tenant1' }) expected_delete_actions = [{ '_op_type': 'delete', '_id': 'id_for_routing_plugin-fake1', '_routing': 'tenant1' }] mock_bulk.assert_called_once_with(client=plugin.engine, index=plugin.alias_name_listener, doc_type=plugin.document_type, actions=expected_delete_actions)
def index_helper(self): if not getattr(self, '_index_helper', None): self._index_helper = helper.IndexingHelper(self) return self._index_helper