def test_parent_child_mapping(self, mock_use_doc_vals): mock_use_doc_vals.return_value = False mock_engine = mock.Mock() parent_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) child_plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) child_plugin.register_parent(parent_plugin) parent_plugin.setup_index_mapping(index_name='fake') # Testing a couple of things; that expected_calls = [ mock.call( index='fake', doc_type='fake-child', body={ '_parent': {'type': 'fake-simple'}, 'properties': { 'updated_at': {'type': 'date'}, 'id': {'type': 'string', 'index': 'not_analyzed'}, 'parent_id': {'type': 'string', 'index': 'not_analyzed'}, ROLE_USER_FIELD: {'include_in_all': False, 'type': 'string', 'index': 'not_analyzed'} } }), mock.call( index='fake', doc_type='fake-simple', body={ 'properties': { 'updated_at': {'type': 'date'}, 'id': {'type': 'string', 'index': 'not_analyzed'}, ROLE_USER_FIELD: {'include_in_all': False, 'type': 'string', 'index': 'not_analyzed'} } }) ] mock_engine.indices.put_mapping.assert_has_calls(expected_calls) # Also test explicitly setting _parent on the child, which should # result in the same mapping mock_engine.reset_mock() mock_engine.indices.put_mapping.assert_not_called() child_mapping = child_plugin.get_mapping() # This mapping matches what would be assigned automatically def explicit_parent_mapping(_self): child_mapping['_parent'] = {'type': 'fake-simple'} return child_mapping child_plugin.get_mapping = types.MethodType(explicit_parent_mapping, child_plugin) parent_plugin.setup_index_mapping(index_name='fake') mock_engine.indices.put_mapping.assert_has_calls(expected_calls)
def test_child_only_separated(self, mock_role_separated): """Test where the child (and not the parent) is separated. Expect two documents with the same parent """ mock_role_separated.return_value = True mock_engine = mock.Mock() plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) parent_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) plugin.register_parent(parent_plugin) indexing_helper = helper.IndexingHelper(plugin) bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' child_docs = copy.deepcopy(fake_plugins.CHILD_DATA) with mock.patch(bulk_name) as mock_bulk: indexing_helper.save_documents(child_docs) self.assertEqual(1, len(mock_bulk.call_args_list)) actions = list(mock_bulk.call_args_list[0][1]['actions']) expected_admin_doc = copy.deepcopy(child_docs[0]) expected_admin_doc[ROLE_USER_FIELD] = 'admin' expected_user_doc = copy.deepcopy(child_docs[0]) expected_user_doc[ROLE_USER_FIELD] = 'user' expected_actions = [ {'_op_type': 'index', '_id': 'child1_ADMIN', '_source': expected_admin_doc, '_parent': 'simple1'}, {'_op_type': 'index', '_id': 'child1_USER', '_source': expected_user_doc, '_parent': 'simple1'} ] self.assertEqual(expected_actions, list(actions))
def test_prepare_index(self, mock_mapping): """Verify Indexbase.prepare_index(). The method will verify that all non-analyzed mapping fields that are raw, are truly marked as raw. This applies to any children plugins. There should not be any exceptions raised. In addition, the index mappings and settings are created at this time. Since we have separate unit tests for verifying the index mappings and index settings functionality, we will verify only that these methods are called. """ mock_engine = mock.Mock() # Test #1: Plugin with no children, good "raw" mapping field. plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) with mock.patch.object(plugin, 'get_mapping') as mock_map: mock_map.return_value = {"properties": { "id": {"type": "string", "index": "not_analyzed"}, "name": {"type": "string", "fields": { "raw": {"type": "string", "index": "not_analyzed"} }}}} plugin.prepare_index('fake') mock_mapping.assert_called_once_with(index_name='fake') # Test #2: Plugin with no children, bad "raw" mapping field. mock_mapping.reset_mock() plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) with mock.patch.object(plugin, 'get_mapping') as mock_map: mock_map.return_value = {"properties": { "id": {"type": "string", "index": "not_analyzed"}, "name": {"type": "string"}}} message = ("Field 'name' for searchlight-listener/fake-simple " "must contain a subfield whose name is 'raw' for " "sorting.") self.assertRaisesRegex(Exception, message, plugin.prepare_index, index_name='fake') mock_mapping.assert_not_called() # Test #3: Plugin with two children. No "raw" mapping fields. mock_mapping.reset_mock() parent_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) child1_plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) child1_plugin.register_parent(parent_plugin) child2_plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) child2_plugin.register_parent(parent_plugin) parent_plugin.prepare_index('fake') mock_mapping.assert_called_once_with(index_name='fake')
def test_delete_children_non_role_separated(self): mock_engine = mock.Mock() plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) parent_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) plugin.register_parent(parent_plugin) indexing_helper = helper.IndexingHelper(plugin) scan_name = 'searchlight.elasticsearch.plugins.helper.helpers.scan' bulk_name = 'searchlight.elasticsearch.plugins.helper.helpers.bulk' mock_scan_data = [ { '_id': '1', 'fields': { '_parent': 'p1' } }, ] with mock.patch(scan_name, return_value=mock_scan_data) as mock_scan: with mock.patch(bulk_name) as mock_bulk: indexing_helper.delete_documents_with_parent('p1') parent_type = plugin.parent_plugin_type() expected_scan_query = { 'fields': ['_parent', '_routing'], 'query': { 'term': { '_parent': '%s#%s' % (parent_type, 'p1') } } } mock_scan.assert_called_with(client=plugin.engine, index=plugin.alias_name_listener, doc_type=plugin.document_type, query=expected_scan_query) expected_delete_actions = [{ '_op_type': 'delete', '_id': '1', '_parent': 'p1' }] mock_bulk.assert_called_with(client=plugin.engine, index=plugin.alias_name_listener, doc_type=plugin.document_type, actions=expected_delete_actions)
def test_manage_type_glob(self, mock_get_plugins, mock_utcnow): mock_utcnow.return_value = datetime.datetime(year=2016, month=3, day=3) expected_index_name = 'searchlight-2016_03_03_00_00_00' simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection) child_plugin = fake_plugins.FakeChildPlugin(self.elastic_connection) non_role_plugin = fake_plugins.NonRoleSeparatedPlugin( self.elastic_connection) child_plugin.register_parent(simple_plugin) mock_get_plugins.return_value = { plugin.get_document_type(): test_utils.StevedoreMock(plugin) for plugin in (simple_plugin, child_plugin, non_role_plugin) } index_command = manage.IndexCommands() # Expect this to match simple plugin and child plugin, but not # non_role_plugin. Patch the index->index function call since it won't # find any data, and we want to check it's called correctly with mock.patch.object(index_command, '_es_reindex_worker') as patch_es_reindex: try: # Use two wildcard matches index_command.sync(_type='fake-sim*,fake-chi*', force=True) es_results = self._get_all_elasticsearch_docs() es_hits = self._get_hit_source(es_results) finally: es_utils.delete_index(expected_index_name) patch_es_reindex.assert_called_with( {non_role_plugin.get_document_type(): non_role_plugin}, [('searchlight', 'searchlight-search', 'searchlight-listener')], {'searchlight': expected_index_name} ) expected = ['simple1', 'child1'] self.assertEqual(len(expected), len(es_hits)) self.assertEqual( set(expected), set(hit['_id'] for hit in es_results['hits']['hits']))
def test_invalid_parent_mapping(self): mock_engine = mock.Mock() parent_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) child_plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) child_plugin.register_parent(parent_plugin) child_mapping = child_plugin.get_mapping() def bad_parent_mapping(_self): child_mapping['_parent'] = {'type': 'this is not my parent'} return child_mapping # Now have the child's mapping include a bad _parent value child_plugin.get_mapping = types.MethodType(bad_parent_mapping, child_plugin) expected_error = ("Mapping for 'fake-child' contains a _parent 'this " "is not my parent' that doesn't match 'fake-simple'") self.assertRaisesRegexp(exception.IndexingException, expected_error, parent_plugin.setup_index_mapping, index_name='fake')
def test_manage(self, mock_get_plugins, mock_utcnow): """Test that manage index sync works from end to end. Uses fake plugins because it avoids having to fake service data and is less dependent on functional tests for each service plugin. """ mock_utcnow.return_value = datetime.datetime(year=2016, month=1, day=1) expected_index_name = 'searchlight-2016_01_01_00_00_00' simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection) child_plugin = fake_plugins.FakeChildPlugin(self.elastic_connection) non_role_plugin = fake_plugins.NonRoleSeparatedPlugin( self.elastic_connection) child_plugin.register_parent(simple_plugin) mock_get_plugins.return_value = { plugin.get_document_type(): test_utils.StevedoreMock(plugin) for plugin in (simple_plugin, child_plugin, non_role_plugin) } index_command = manage.IndexCommands() # The fake plugins all have hardcoded data for get_objects that will # be indexed. Use force=True to avoid the 'are you sure?' prompt try: index_command.sync(force=True) es_results = self._get_all_elasticsearch_docs() es_hits = self._get_hit_source(es_results) finally: es_utils.delete_index(expected_index_name) self.assertEqual(expected_index_name, es_results['hits']['hits'][0]['_index']) expected = ['simple1', 'child1', 'non-role-fake1', 'non-role-fake2'] self.assertEqual(len(expected), len(es_hits)) self.assertEqual( set(expected), set(hit['_id'] for hit in es_results['hits']['hits']))
def test_get_facets(self): mock_engine = mock.Mock() simple_plugin = fake_plugins.FakeSimplePlugin(es_engine=mock_engine) simple_plugin.engine = mock_engine child_plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) child_plugin.engine = mock_engine mock_engine.search.return_value = { 'aggregations': {}, 'hits': { 'total': 1 } } fake_request = unit_test_utils.get_fake_request() meta_mapping = { 'properties': { 'reference_id_1': { 'type': 'string', 'index': 'not_analyzed' }, 'some_key': { 'type': 'integer' }, 'nested': { 'type': 'nested', 'properties': { "some_key": { 'type': 'string' }, 'reference_id_2': { 'type': 'string', 'index': 'not_analyzed' } } } }, "_meta": { "reference_id_1": { "resource_type": "OS::Glance::Image" }, "nested.reference_id_2": { "resource_type": "OS::Cinder::Snapshot" } } } # Test resource_types without parent with mock.patch.object(simple_plugin, 'get_mapping', return_value=meta_mapping): facets, doc_count = simple_plugin.get_facets(fake_request.context) expected = [{ "type": "string", "name": "reference_id_1", "resource_type": "OS::Glance::Image" }, { "type": "string", "name": "nested.reference_id_2", "resource_type": "OS::Cinder::Snapshot", "nested": True, }, { "type": "string", "name": "nested.some_key", "nested": True }, { "type": "integer", "name": "some_key", }] expected_list = sorted(expected, key=lambda k: k['name']) actual_list = sorted(facets, key=lambda k: k['name']) self.assertEqual(expected_list, actual_list) # Test resource_types with parent meta_mapping = { 'properties': { 'parent_id': { 'type': 'string', 'index': 'not_analyzed' } }, "_meta": { "parent_id": { "resource_type": child_plugin.parent_plugin_type() } } } with mock.patch.object(child_plugin, 'get_mapping', return_value=meta_mapping): facets, doc_count = child_plugin.get_facets(fake_request.context) expected = [{ "type": "string", "name": "parent_id", "resource_type": child_plugin.parent_plugin_type(), "parent": True }] self.assertEqual(expected, facets) # Test resource_types with parent and no explicit meta info meta_mapping.pop('_meta') with mock.patch.object(child_plugin, 'get_mapping', return_value=meta_mapping): facets, doc_count = child_plugin.get_facets(fake_request.context) expected = [{ "type": "string", "name": "parent_id", "resource_type": child_plugin.parent_plugin_type(), "parent": True }] self.assertEqual(expected, facets)
def test_delete_unknown_parent(self): mock_engine = mock.Mock() plugin = fake_plugins.FakeSeparatedChildPlugin(es_engine=mock_engine) doc_id = 'aaaa-bbbb' parent_id = '1111-2222' # Mock the delete function because the internals are tested elsewhere # First test with parent and routing with mock.patch.object(plugin.index_helper, 'delete_document') as mock_del: mock_engine.search.return_value = { 'hits': { 'total': 1, 'hits': [{ '_id': doc_id + helper.ADMIN_ID_SUFFIX, '_routing': parent_id, '_parent': parent_id + helper.ADMIN_ID_SUFFIX }] } } plugin.index_helper.delete_document_unknown_parent(doc_id) mock_del.assert_called_with({ '_id': doc_id, '_parent': parent_id, '_routing': parent_id }) # Now no explicit routing with mock.patch.object(plugin.index_helper, 'delete_document') as mock_del: mock_engine.search.return_value = { 'hits': { 'total': 1, 'hits': [{ '_id': doc_id + helper.ADMIN_ID_SUFFIX, '_parent': parent_id + helper.ADMIN_ID_SUFFIX }] } } plugin.index_helper.delete_document_unknown_parent(doc_id) mock_del.assert_called_with({'_id': doc_id, '_parent': parent_id}) # Test no results found with mock.patch.object(plugin.index_helper, 'delete_document') as mock_del: mock_engine.search.return_value = { 'hits': { 'total': 0, 'hits': [] } } plugin.index_helper.delete_document_unknown_parent(doc_id) self.assertEqual(0, mock_del.call_count) # Also test a non-separated plugin mock_engine = mock.Mock() plugin = fake_plugins.FakeChildPlugin(es_engine=mock_engine) with mock.patch.object(plugin.index_helper, 'delete_document') as mock_del: mock_engine.search.return_value = { 'hits': { 'total': 1, 'hits': [{ '_id': doc_id, '_parent': parent_id }] } } plugin.index_helper.delete_document_unknown_parent(doc_id) mock_del.assert_called_with({'_id': doc_id, '_parent': parent_id})