def test_bulk_index_error_handling(self):
"""Check that 404 and 409 errors are appropriately ignored"""
from elasticsearch import helpers
mock_engine = mock.Mock()
plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
with mock.patch(bulk_name) as mock_bulk:
mock_bulk.side_effect = helpers.BulkIndexError(
"1 document(s) failed to index", [{
'delete': {
"_id": "1",
"error": "Some error",
"status": 404,
"exception": helpers.TransportError()
}
}])
indexing_helper.delete_documents([{'_id': '1'}])
self.assertEqual(1, mock_bulk.call_count)
with mock.patch(bulk_name) as mock_bulk:
mock_bulk.side_effect = helpers.BulkIndexError(
"1 document(s) failed to index", [{
'index': {
"_id": "1",
"error": "VersionConflict",
"status": 409
}
}])
indexing_helper.save_documents([{'id': '1'}])
self.assertEqual(1, mock_bulk.call_count)
def test_non_role_separated_save_docs(self):
"""Test that for a plugin that doesn't specify any protected fields,
ids are left alone and there's only one copy of each indexed doc
"""
mock_engine = mock.Mock()
plugin = fake_plugins.NonRoleSeparatedPlugin(es_engine=mock_engine)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
with mock.patch(bulk_name) as mock_bulk:
count = len(plugin.get_objects())
fake_versions = range(1, count + 1)
indexing_helper.save_documents(plugin.get_objects(), fake_versions)
self.assertEqual(1, len(mock_bulk.call_args_list))
actions = list(mock_bulk.call_args_list[0][1]['actions'])
self.assertEqual(2, len(actions))
self.assertEqual(set(['non-role-fake1', 'non-role-fake2']),
set(action['_id'] for action in actions))
self.assertEqual(set(fake_versions),
set(action['_version'] for action in actions))
self.assertEqual(['external'] * 2,
[action['_version_type'] for action in actions])
fake1 = actions[0]['_source']
self.assertEqual(['admin', 'user'], sorted(fake1.pop(ROLE_USER_FIELD)))
self.assertEqual(fake_plugins.NON_ROLE_SEPARATED_DATA[0],
actions[0]['_source'])
def test_region_mapping(self, service_credentials_conf,
resource_plugin_conf, mock_bulk):
mock_engine = mock.Mock()
plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine)
resource_plugin_conf.include_region_name = True
service_credentials_conf.os_region_name = 'test-region'
indexing_helper = helper.IndexingHelper(plugin)
_, mapping = six.next(plugin.get_full_mapping())
self.assertIn('region_name', mapping['properties'])
count = len(plugin.get_objects())
fake_versions = range(1, count + 1)
indexing_helper.save_documents(plugin.get_objects(), fake_versions)
self.assertEqual(1, len(mock_bulk.call_args_list))
actions = list(mock_bulk.call_args_list[0][1]['actions'])
self.assertEqual(['test-region'], actions[0]['_source']['region_name'])
# Test without a region
resource_plugin_conf.include_region_name = False
mock_bulk.reset_mock()
_, mapping = six.next(plugin.get_full_mapping())
self.assertNotIn('region_name', mapping['properties'])
indexing_helper.save_documents(plugin.get_objects(), fake_versions)
actions = list(mock_bulk.call_args_list[0][1]['actions'])
self.assertNotIn('region_name', actions[0]['_source'])
def test_child_only_separated(self, mock_role_separated):
"""Test where the child (and not the parent) is separated. Expect
two documents with the same parent
"""
mock_role_separated.return_value = True
mock_engine = mock.Mock()
plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine)
parent_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine)
plugin.register_parent(parent_plugin)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
child_docs = copy.deepcopy(fake_plugins.CHILD_DATA)
with mock.patch(bulk_name) as mock_bulk:
indexing_helper.save_documents(child_docs)
self.assertEqual(1, len(mock_bulk.call_args_list))
actions = list(mock_bulk.call_args_list[0][1]['actions'])
expected_admin_doc = copy.deepcopy(child_docs[0])
expected_admin_doc[ROLE_USER_FIELD] = 'admin'
expected_user_doc = copy.deepcopy(child_docs[0])
expected_user_doc[ROLE_USER_FIELD] = 'user'
expected_actions = [
{'_op_type': 'index', '_id': 'child1_ADMIN',
'_source': expected_admin_doc, '_parent': 'simple1'},
{'_op_type': 'index', '_id': 'child1_USER',
'_source': expected_user_doc, '_parent': 'simple1'}
]
self.assertEqual(expected_actions, list(actions))
def test_save_parent_only_separated(self, mock_role_separated):
"""Test where the parent document is role separated but this child
is not; it's expected the parent's _USER documents be used for _parent
"""
mock_role_separated.return_value = False
mock_engine = mock.Mock()
plugin = fake_plugins.FakeSeparatedChildPlugin(es_engine=mock_engine)
parent_plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine)
plugin.register_parent(parent_plugin)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
child_docs = copy.deepcopy(fake_plugins.CHILD_DATA)
with mock.patch(bulk_name) as mock_bulk:
indexing_helper.save_documents(child_docs)
self.assertEqual(1, len(mock_bulk.call_args_list))
actions = list(mock_bulk.call_args_list[0][1]['actions'])
expected_doc = copy.deepcopy(child_docs[0])
expected_doc[ROLE_USER_FIELD] = ['user', 'admin']
expected_actions = [
{'_op_type': 'index', '_id': 'child1',
'_source': expected_doc, '_parent': 'simple1_USER'},
]
self.assertEqual(expected_actions, list(actions))
def test_save_child_parent_both_separated(self):
mock_engine = mock.Mock()
plugin = fake_plugins.FakeSeparatedChildPlugin(es_engine=mock_engine)
parent_plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine)
plugin.register_parent(parent_plugin)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
child_docs = copy.deepcopy(fake_plugins.CHILD_DATA)
# First run where both child and parent are role-separated (and thus
# we'd expect two copies of both parent and child with appropriate
# ids linking them)
with mock.patch(bulk_name) as mock_bulk:
indexing_helper.save_documents(child_docs)
self.assertEqual(1, len(mock_bulk.call_args_list))
actions = list(mock_bulk.call_args_list[0][1]['actions'])
expected_admin_doc = copy.deepcopy(child_docs[0])
expected_admin_doc[ROLE_USER_FIELD] = 'admin'
expected_user_doc = copy.deepcopy(child_docs[0])
expected_user_doc[ROLE_USER_FIELD] = 'user'
expected_actions = [
{'_op_type': 'index', '_id': 'child1_ADMIN',
'_source': expected_admin_doc, '_parent': 'simple1_ADMIN'},
{'_op_type': 'index', '_id': 'child1_USER',
'_source': expected_user_doc, '_parent': 'simple1_USER'}
]
self.assertEqual(expected_actions, list(actions))
def test_delete_children_role_separated(self):
mock_engine = mock.Mock()
plugin = fake_plugins.FakeSeparatedChildPlugin(es_engine=mock_engine)
parent_plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine)
plugin.register_parent(parent_plugin)
indexing_helper = helper.IndexingHelper(plugin)
scan_name = 'searchlight.elasticsearch.plugins.helper.helpers.scan'
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
mock_scan_data = [{
'_id': '1_ADMIN',
'fields': {
'_parent': 'p1_ADMIN'
}
}, {
'_id': '1_USER',
'fields': {
'_parent': 'p1_USER'
}
}]
with mock.patch(scan_name, return_value=mock_scan_data) as mock_scan:
with mock.patch(bulk_name) as mock_bulk:
indexing_helper.delete_documents_with_parent('p1')
parent_type = plugin.parent_plugin_type()
base_full_parent_type = '%s#%s' % (parent_type, 'p1')
expected_scan_query = {
'fields': ['_parent', '_routing'],
'query': {
'terms': {
'_parent': [
base_full_parent_type + '_ADMIN',
base_full_parent_type + '_USER'
]
}
}
}
mock_scan.assert_called_with(client=plugin.engine,
index=plugin.alias_name_listener,
doc_type=plugin.document_type,
query=expected_scan_query)
expected_delete_actions = [{
'_op_type': 'delete',
'_id': '1_ADMIN',
'_parent': 'p1_ADMIN'
}, {
'_op_type': 'delete',
'_id': '1_USER',
'_parent': 'p1_USER'
}]
mock_bulk.assert_called_with(client=plugin.engine,
index=plugin.alias_name_listener,
doc_type=plugin.document_type,
actions=expected_delete_actions)
def test_non_role_separated_delete(self):
"""Test that deletion for a role-separated plugin deletes the doc"""
mock_engine = mock.Mock()
plugin = fake_plugins.NonRoleSeparatedPlugin(es_engine=mock_engine)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
with mock.patch(bulk_name) as mock_bulk:
indexing_helper.delete_document({'_id': 'non-role-fake1'})
expected_delete_actions = [{
'_op_type': 'delete',
'_id': 'non-role-fake1'
}]
mock_bulk.assert_called_once_with(client=plugin.engine,
index=plugin.alias_name_listener,
doc_type=plugin.document_type,
actions=expected_delete_actions)
def test_role_separated_save_docs(self):
"""Test admin only fields are correctly removed from serialization
and that alternate _id values are used
"""
mock_engine = mock.Mock()
plugin = fake_plugins.RoleSeparatedPlugin(es_engine=mock_engine)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
with mock.patch(bulk_name) as mock_bulk:
count = len(plugin.get_objects())
fake_versions = range(1, count + 1)
indexing_helper.save_documents(plugin.get_objects(), fake_versions)
self.assertEqual(1, len(mock_bulk.call_args_list))
actions = list(mock_bulk.call_args_list[0][1]['actions'])
self.assertEqual(4, len(actions))
self.assertEqual(
set([
'role-fake1_ADMIN', 'role-fake2_ADMIN', 'role-fake1_USER',
'role-fake2_USER'
]), set(action['_id'] for action in actions))
self.assertEqual(set(fake_versions),
set(action['_version'] for action in actions))
self.assertEqual(['external'] * 4,
[action['_version_type'] for action in actions])
# This plugin filters on admin_wildcard_* and admin_specific
fake1_admin = list(
filter(lambda a: a['_id'] == 'role-fake1_ADMIN',
actions))[0]['_source']
self.assertEqual('role-fake1', fake1_admin['id'])
self.assertIn('public_field', fake1_admin)
self.assertIn('admin_wildcard_this', fake1_admin)
self.assertEqual('admin', fake1_admin[ROLE_USER_FIELD])
fake1_user = list(
filter(lambda a: a['_id'] == 'role-fake1_USER',
actions))[0]['_source']
self.assertEqual('role-fake1', fake1_user['id'])
self.assertIn('public_field', fake1_user)
self.assertNotIn('admin_wildcard_this', fake1_user)
self.assertNotIn('admin_specific', fake1_user)
self.assertEqual('user', fake1_user[ROLE_USER_FIELD])
def test_routing_save_docs(self):
"""Test that for a plugin that specifies routing_id field will
end up with "_routing" set while indexing.
"""
mock_engine = mock.Mock()
plugin = fake_plugins.FakeSimpleRoutingPlugin(es_engine=mock_engine)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
with mock.patch(bulk_name) as mock_bulk:
count = len(plugin.get_objects())
fake_versions = range(1, count + 1)
indexing_helper.save_documents(plugin.get_objects(), fake_versions)
self.assertEqual(1, len(mock_bulk.call_args_list))
actions = list(mock_bulk.call_args_list[0][1]['actions'])
# '_routing' is added to action only if set in the plugin property
# FakeSimpleRoutingPlugin has it defined.
self.assertIs(True, '_routing' in actions[0])
self.assertEqual('tenant1', actions[0]['_routing'])
def test_routing_delete(self):
"""Test that deletion for a routing based plugin deletes docs"""
mock_engine = mock.Mock()
plugin = fake_plugins.FakeSimpleRoutingPlugin(es_engine=mock_engine)
indexing_helper = helper.IndexingHelper(plugin)
bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk'
with mock.patch(bulk_name) as mock_bulk:
indexing_helper.delete_document({
'_id': 'id_for_routing_plugin-fake1',
'_routing': 'tenant1'
})
expected_delete_actions = [{
'_op_type': 'delete',
'_id': 'id_for_routing_plugin-fake1',
'_routing': 'tenant1'
}]
mock_bulk.assert_called_once_with(client=plugin.engine,
index=plugin.alias_name_listener,
doc_type=plugin.document_type,
actions=expected_delete_actions)
def index_helper(self):
if not getattr(self, '_index_helper', None):
self._index_helper = helper.IndexingHelper(self)
return self._index_helper
