def _setup_es_indexes(_es_client):
"""Sets up ES and makes the client available."""
# Create models in the test index
for search_app in get_search_apps():
# Clean up in case of any aborted test runs
index_name = search_app.es_model.get_target_index_name()
read_alias = search_app.es_model.get_read_alias()
write_alias = search_app.es_model.get_write_alias()
if index_exists(index_name):
delete_index(index_name)
if alias_exists(read_alias):
delete_alias(read_alias)
if alias_exists(write_alias):
delete_alias(write_alias)
# Create indices and aliases
alias_names = (read_alias, write_alias)
create_index(index_name,
search_app.es_model._doc_type.mapping,
alias_names=alias_names)
yield _es_client
for search_app in get_search_apps():
delete_index(search_app.es_model.get_target_index_name())
def _es_session(_es_client):
"""
Session-scoped fixture that creates Elasticsearch indexes that persist for the entire test
session.
"""
# Create models in the test index
for search_app in get_search_apps():
# Clean up in case of any aborted test runs
index_name = search_app.es_model.get_target_index_name()
read_alias = search_app.es_model.get_read_alias()
write_alias = search_app.es_model.get_write_alias()
if index_exists(index_name):
delete_index(index_name)
if alias_exists(read_alias):
delete_alias(read_alias)
if alias_exists(write_alias):
delete_alias(write_alias)
# Create indices and aliases
alias_names = (read_alias, write_alias)
create_index(index_name,
search_app.es_model._doc_type.mapping,
alias_names=alias_names)
yield _es_client
for search_app in get_search_apps():
delete_index(search_app.es_model.get_target_index_name())
def _perform_migration(search_app):
app_name = search_app.name
es_model = search_app.es_model
logger.info(f'Migrating the {app_name} search app')
read_alias_name = es_model.get_read_alias()
write_alias_name = es_model.get_write_alias()
new_index_name = es_model.get_target_index_name()
current_read_indices, current_write_index = es_model.get_read_and_write_indices(
)
if current_write_index not in current_read_indices:
raise DataHubException(
'Cannot migrate Elasticsearch index with a read alias referencing '
'a different index to the write alias', )
logger.info(f'Updating aliases for the {app_name} search app')
create_index(new_index_name, es_model._doc_type.mapping)
with start_alias_transaction() as alias_transaction:
alias_transaction.associate_indices_with_alias(read_alias_name,
[new_index_name])
alias_transaction.associate_indices_with_alias(write_alias_name,
[new_index_name])
alias_transaction.dissociate_indices_from_alias(
write_alias_name, [current_write_index])
_schedule_resync(search_app)
def set_up_index_and_aliases(cls):
"""Creates the index and aliases for this model if they don't already exist."""
if not alias_exists(cls.get_write_alias()):
index_name = cls.get_target_index_name()
alias_names = (cls.get_write_alias(), cls.get_read_alias())
create_index(index_name, cls._doc_type.mapping, alias_names=alias_names)
return True
# Should not normally happen
if not alias_exists(cls.get_read_alias()):
logger.warning(
f'Missing read alias {cls.get_read_alias()} detected, recreating the alias...',
)
associate_index_with_alias(cls.get_read_alias(), cls.get_write_index())
return False
def set_up_index_and_aliases(cls, force_update_mapping=False):
"""
Creates the index and aliases for this model if they don't already exist.
If force_update_mapping is True and the write alias already exists, an attempt
is made to update to update the existing mapping in place.
"""
if not alias_exists(cls.get_write_alias()):
index_name = cls.get_target_index_name()
alias_names = (cls.get_write_alias(), cls.get_read_alias())
create_index(index_name,
cls._doc_type.mapping,
alias_names=alias_names)
elif force_update_mapping:
cls.init(cls.get_write_alias())
# Should not normally happen
if not alias_exists(cls.get_read_alias()):
logger.warning(
f'Missing read alias {cls.get_read_alias()} detected, recreating '
f'the alias...', )
associate_index_with_alias(cls.get_read_alias(),
cls.get_write_index())
def test_creates_index(monkeypatch, mock_es_client):
"""Test creates_index()."""
monkeypatch.setattr(
'django.conf.settings.ES_INDEX_SETTINGS',
{
'testsetting1': 'testval1',
},
)
mapping = Mapping('test-mapping')
mapping.field('test-field', Keyword())
index = 'test-index'
connection = mock_es_client.return_value
elasticsearch.create_index(index,
mapping,
alias_names=('alias1', 'alias2'))
connection.indices.create.assert_called_once_with(
index='test-index',
body={
'settings': {
'testsetting1': 'testval1',
'analysis': {
'analyzer': {
'trigram_analyzer': {
'tokenizer': 'trigram',
'char_filter': ['special_chars'],
'filter': ['lowercase'],
'type': 'custom',
},
'english_analyzer': {
'tokenizer':
'standard',
'filter': [
'english_possessive_stemmer',
'lowercase',
'english_stop',
'english_stemmer',
],
'type':
'custom',
},
},
'tokenizer': {
'trigram': {
'min_gram': 3,
'max_gram': 3,
'token_chars': ('letter', 'digit'),
'type': 'nGram',
},
},
'char_filter': {
'special_chars': {
'mappings': ('-=>', ),
'type': 'mapping',
},
},
'filter': {
'english_possessive_stemmer': {
'language': 'possessive_english',
'type': 'stemmer',
},
'english_stop': {
'stopwords': '_english_',
'type': 'stop',
},
'english_stemmer': {
'language': 'english',
'type': 'stemmer',
},
},
},
},
'aliases': {
'alias1': {},
'alias2': {},
},
'mappings': {
'test-mapping': {
'properties': {
'test-field': {
'type': 'keyword',
},
},
},
},
},
)