def _setup_es_indexes(_es_client): """Sets up ES and makes the client available.""" # Create models in the test index for search_app in get_search_apps(): # Clean up in case of any aborted test runs index_name = search_app.es_model.get_target_index_name() read_alias = search_app.es_model.get_read_alias() write_alias = search_app.es_model.get_write_alias() if index_exists(index_name): delete_index(index_name) if alias_exists(read_alias): delete_alias(read_alias) if alias_exists(write_alias): delete_alias(write_alias) # Create indices and aliases alias_names = (read_alias, write_alias) create_index(index_name, search_app.es_model._doc_type.mapping, alias_names=alias_names) yield _es_client for search_app in get_search_apps(): delete_index(search_app.es_model.get_target_index_name())
def _es_session(_es_client): """ Session-scoped fixture that creates Elasticsearch indexes that persist for the entire test session. """ # Create models in the test index for search_app in get_search_apps(): # Clean up in case of any aborted test runs index_name = search_app.es_model.get_target_index_name() read_alias = search_app.es_model.get_read_alias() write_alias = search_app.es_model.get_write_alias() if index_exists(index_name): delete_index(index_name) if alias_exists(read_alias): delete_alias(read_alias) if alias_exists(write_alias): delete_alias(write_alias) # Create indices and aliases alias_names = (read_alias, write_alias) create_index(index_name, search_app.es_model._doc_type.mapping, alias_names=alias_names) yield _es_client for search_app in get_search_apps(): delete_index(search_app.es_model.get_target_index_name())
def _perform_migration(search_app): app_name = search_app.name es_model = search_app.es_model logger.info(f'Migrating the {app_name} search app') read_alias_name = es_model.get_read_alias() write_alias_name = es_model.get_write_alias() new_index_name = es_model.get_target_index_name() current_read_indices, current_write_index = es_model.get_read_and_write_indices( ) if current_write_index not in current_read_indices: raise DataHubException( 'Cannot migrate Elasticsearch index with a read alias referencing ' 'a different index to the write alias', ) logger.info(f'Updating aliases for the {app_name} search app') create_index(new_index_name, es_model._doc_type.mapping) with start_alias_transaction() as alias_transaction: alias_transaction.associate_indices_with_alias(read_alias_name, [new_index_name]) alias_transaction.associate_indices_with_alias(write_alias_name, [new_index_name]) alias_transaction.dissociate_indices_from_alias( write_alias_name, [current_write_index]) _schedule_resync(search_app)
def set_up_index_and_aliases(cls): """Creates the index and aliases for this model if they don't already exist.""" if not alias_exists(cls.get_write_alias()): index_name = cls.get_target_index_name() alias_names = (cls.get_write_alias(), cls.get_read_alias()) create_index(index_name, cls._doc_type.mapping, alias_names=alias_names) return True # Should not normally happen if not alias_exists(cls.get_read_alias()): logger.warning( f'Missing read alias {cls.get_read_alias()} detected, recreating the alias...', ) associate_index_with_alias(cls.get_read_alias(), cls.get_write_index()) return False
def set_up_index_and_aliases(cls, force_update_mapping=False): """ Creates the index and aliases for this model if they don't already exist. If force_update_mapping is True and the write alias already exists, an attempt is made to update to update the existing mapping in place. """ if not alias_exists(cls.get_write_alias()): index_name = cls.get_target_index_name() alias_names = (cls.get_write_alias(), cls.get_read_alias()) create_index(index_name, cls._doc_type.mapping, alias_names=alias_names) elif force_update_mapping: cls.init(cls.get_write_alias()) # Should not normally happen if not alias_exists(cls.get_read_alias()): logger.warning( f'Missing read alias {cls.get_read_alias()} detected, recreating ' f'the alias...', ) associate_index_with_alias(cls.get_read_alias(), cls.get_write_index())
def test_creates_index(monkeypatch, mock_es_client): """Test creates_index().""" monkeypatch.setattr( 'django.conf.settings.ES_INDEX_SETTINGS', { 'testsetting1': 'testval1', }, ) mapping = Mapping('test-mapping') mapping.field('test-field', Keyword()) index = 'test-index' connection = mock_es_client.return_value elasticsearch.create_index(index, mapping, alias_names=('alias1', 'alias2')) connection.indices.create.assert_called_once_with( index='test-index', body={ 'settings': { 'testsetting1': 'testval1', 'analysis': { 'analyzer': { 'trigram_analyzer': { 'tokenizer': 'trigram', 'char_filter': ['special_chars'], 'filter': ['lowercase'], 'type': 'custom', }, 'english_analyzer': { 'tokenizer': 'standard', 'filter': [ 'english_possessive_stemmer', 'lowercase', 'english_stop', 'english_stemmer', ], 'type': 'custom', }, }, 'tokenizer': { 'trigram': { 'min_gram': 3, 'max_gram': 3, 'token_chars': ('letter', 'digit'), 'type': 'nGram', }, }, 'char_filter': { 'special_chars': { 'mappings': ('-=>', ), 'type': 'mapping', }, }, 'filter': { 'english_possessive_stemmer': { 'language': 'possessive_english', 'type': 'stemmer', }, 'english_stop': { 'stopwords': '_english_', 'type': 'stop', }, 'english_stemmer': { 'language': 'english', 'type': 'stemmer', }, }, }, }, 'aliases': { 'alias1': {}, 'alias2': {}, }, 'mappings': { 'test-mapping': { 'properties': { 'test-field': { 'type': 'keyword', }, }, }, }, }, )