def refresh_index(self):
"""
Refreshes an index.
https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html
"""
ElasticsearchUtils.refresh_index(self.es, self.index)
def refresh_index(self):
"""
Refreshes an index.
https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html
"""
for index in registry.get_indices():
ElasticsearchUtils.refresh_index(self.es, index._name) # pylint: disable=protected-access
def test_handle(self, elasticsearch_dsl_default_connection):
""" Verify the command removes all but the newest indexes. """
# Use now as initial time, so indexes are created AFTER the current index so expected values are accurate
initial_time = datetime.datetime.now()
# Create 2 more indexes than we expect to exist after removal
for number in range(
1, settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + 2):
current_time = initial_time + datetime.timedelta(seconds=number)
freezer = freeze_time(current_time)
freezer.start()
for index in registry.get_indices():
ElasticsearchUtils.create_index(index)
freezer.stop()
# Prune indexes and confirm the right indexes are removed
call_command('remove_unused_indexes')
indices_client = elasticsearch_dsl_default_connection.indices
for index in registry.get_indices():
# pylint: disable=protected-access
index_alias = ElasticsearchUtils.get_alias_by_index_name(
index._name)
current_alias = indices_client.get_alias(name=index_alias)
indexes_to_keep = current_alias.keys()
# check that we keep the current indexes, which we don't want removed
all_indexes = self.get_current_index_names(
indices_client=indices_client, index_prefix=index_alias)
assert set(all_indexes).issuperset(set(indexes_to_keep))
# check that other indexes are removed, excepting those that don't hit the retention limit
expected_count = settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + len(
indexes_to_keep)
assert len(all_indexes) == expected_count
# Attempt to prune indexes again and confirm that no indexes are removed
call_command('remove_unused_indexes')
for index in registry.get_indices():
# check that we keep the current indexes, which we don't want removed
# pylint: disable=protected-access
index_alias = ElasticsearchUtils.get_alias_by_index_name(
index._name)
all_indexes = self.get_current_index_names(
indices_client=indices_client, index_prefix=index_alias)
current_alias = indices_client.get_alias(name=index_alias)
indexes_to_keep = current_alias.keys()
assert set(all_indexes).issuperset(set(indexes_to_keep))
# check that index count remains the same as before
expected_count = settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + len(
indexes_to_keep)
assert len(all_indexes) == expected_count
# Cleanup indexes created by this test
for index in registry.get_indices():
# pylint: disable=protected-access
indices_client.delete(index=index._name + '_*')
def handle(self, *args, **options):
host = settings.HAYSTACK_CONNECTIONS['default']['URL']
alias = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host)
es = Elasticsearch(host)
logger.info('...success!')
ElasticsearchUtils.create_alias_and_index(es, alias)
def handle(self, *args, **options):
host = settings.HAYSTACK_CONNECTIONS['default']['URL']
alias = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
logger.info(
'Attempting to establish initial connection to Elasticsearch host [%s]...',
host)
es = Elasticsearch(host)
logger.info('...success!')
ElasticsearchUtils.create_alias_and_index(es, alias)
def _update(self, models, options):
"""
Update indices with sanity check.
Will be created a new index and populate with data.
The index will be masked with previous one to prevent missing data.
"""
alias_mappings = []
for document in registry.get_documents(models):
# pylint: disable=protected-access
index = document._index
record_count = self.get_record_count(document)
alias, new_index_name = self.prepare_backend_index(index)
alias_mappings.append(
AliasMapper(document, index, new_index_name, alias,
record_count))
# Set the alias (from settings) to the timestamped catalog.
run_attempts = 0
indexes_pending = {
key: ''
for key in [x.new_index_name for x in alias_mappings]
}
conn = get_connection()
while indexes_pending and run_attempts < 1: # Only try once, as retries gave buggy results. See VAN-391
run_attempts += 1
self._populate(models, options)
for doc, __, new_index_name, alias, record_count in alias_mappings:
# Run a sanity check to ensure we aren't drastically changing the
# index, which could be indicative of a bug.
if new_index_name in indexes_pending and not options.get(
'disable_change_limit', False):
record_count_is_sane, index_info_string = self.sanity_check_new_index(
run_attempts, doc, new_index_name, record_count)
if record_count_is_sane:
ElasticsearchUtils.set_alias(conn, alias,
new_index_name)
indexes_pending.pop(new_index_name, None)
else:
indexes_pending[new_index_name] = index_info_string
else:
ElasticsearchUtils.set_alias(conn, alias, new_index_name)
indexes_pending.pop(new_index_name, None)
for index_alias_mapper in alias_mappings:
index_alias_mapper.registered_index._name = index_alias_mapper.alias # pylint: disable=protected-access
if indexes_pending:
raise CommandError(
'Sanity check failed for the new index(es): {}'.format(
indexes_pending))
return True
def handle(self, *args, **options):
for backend_name, host_cong in settings.ELASTICSEARCH_DSL.items():
logger.info(
'Attempting to establish initial connection to Elasticsearch host [%s]...',
host_cong['hosts'])
es_connection = get_connection(backend_name)
es_connection.ping()
logger.info('...success!')
for index in registry.get_indices():
ElasticsearchUtils.create_alias_and_index(
es_connection, index, backend_name)
def _create(self, models, options):
for backend in self.backends:
for index in registry.get_indices(models):
created_index_info = ElasticsearchUtils.create_index(
index, backend)
es_connection = get_connection(backend)
self.stdout.write(
'Creating index "{1}".\nSet alias "{0}" for index "{1}".'.
format(created_index_info.alias, created_index_info.name))
ElasticsearchUtils.set_alias(es_connection,
created_index_info.alias,
created_index_info.name)
def prepare_backend_index(self, registered_index, backend='default'):
"""
Prepares an index that will be used to store data by the backend.
Args:
registered_index (Index): django elasticsearch index instance.
backend (ElasticsearchSearchBackend): Backend to update.
Returns:
(tuple): tuple containing:
alias(str): Recommended alias for the new index.
index_name(str): Name of the newly-created index.
"""
timestamp = datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S')
# pylint: disable=protected-access
computed_alias = ElasticsearchUtils.get_alias_by_index_name(
registered_index._name)
new_index_name = '{alias}_{timestamp}'.format(alias=computed_alias,
timestamp=timestamp)
registered_index._name = new_index_name
self.stdout.write("Creating index '{}'".format(registered_index._name))
registered_index.create(using=backend)
return computed_alias, new_index_name
def test_handle(self):
""" Verify the command removes all but the newest indexes. """
# Create initial index with alias
ElasticsearchUtils.create_alias_and_index(
es_connection=self.backend.conn, alias=self.backend.index_name)
# Use now as initial time, so indexes are created AFTER the current index so expected values are accurate
initial_time = datetime.datetime.now()
# Create 2 more indexes than we expect to exist after removal
for number in range(1, settings.HAYSTACK_INDEX_RETENTION_LIMIT + 2):
current_time = initial_time + datetime.timedelta(seconds=number)
freezer = freeze_time(current_time)
freezer.start()
ElasticsearchUtils.create_index(es_connection=self.backend.conn,
prefix=self.backend.index_name)
freezer.stop()
# Prune indexes and confirm the right indexes are removed
call_command('remove_unused_indexes')
current_alias_name = self.backend.index_name
indices_client = self.backend.conn.indices
current_alias = indices_client.get_alias(name=current_alias_name)
indexes_to_keep = current_alias.keys()
# check that we keep the current indexes, which we don't want removed
all_indexes = self.get_current_index_names(
indices_client=indices_client,
index_prefix=self.backend.index_name)
assert set(all_indexes).issuperset(set(indexes_to_keep))
# check that other indexes are removed, excepting those that don't hit the retention limit
expected_count = settings.HAYSTACK_INDEX_RETENTION_LIMIT + len(
indexes_to_keep)
assert len(all_indexes) == expected_count
# Attempt to prune indexes again and confirm that no indexes are removed
call_command('remove_unused_indexes')
# check that we keep the current indexes, which we don't want removed
all_indexes = self.get_current_index_names(
indices_client=indices_client,
index_prefix=self.backend.index_name)
assert set(all_indexes).issuperset(set(indexes_to_keep))
# check that index count remains the same as before
assert len(all_indexes) == expected_count
def get_current_index_names(indices_client, index_prefix):
all_indexes = indices_client.get('*').keys()
all_current_indexes = [
index_name for index_name in all_indexes
if ElasticsearchUtils.get_alias_by_index_name(index_name) ==
index_prefix
]
return all_current_indexes
def haystack_default_connection(haystack_add_xdist_suffix_to_index_name): # pylint: disable=redefined-outer-name,unused-argument
skip_if_no_django()
backend = haystack_connections['default'].get_backend()
# Force Haystack to update the mapping for the index
backend.setup_complete = False
es = backend.conn
index_name = backend.index_name
ElasticsearchUtils.delete_index(es, index_name)
ElasticsearchUtils.create_alias_and_index(es, index_name)
ElasticsearchUtils.refresh_index(es, index_name)
yield backend
ElasticsearchUtils.delete_index(es, index_name)
def prepare_backend_index(self, backend):
"""
Prepares an index that will be used to store data by the backend.
Args:
backend (ElasticsearchSearchBackend): Backend to update.
Returns:
(tuple): tuple containing:
alias(str): Recommended alias for the new index.
index_name(str): Name of the newly-created index.
"""
alias = backend.index_name
index_name = ElasticsearchUtils.create_index(backend.conn, alias)
backend.index_name = index_name
return alias, index_name
def get_serializer_class_by_instance(self, instance):
if isinstance(instance, EmptySearch):
serializer_class = serializers.Serializer
else:
instance_serializers = {
# pylint: disable=protected-access
document._index._name: serializer
for document, serializer in self.Meta.serializers.items()
}
index_name = instance.meta['index']
index_alias = ElasticsearchUtils.get_alias_by_index_name(
index_name)
serializer_class = instance_serializers.get(index_alias, None)
if not serializer_class:
raise ImproperlyConfigured(
'Could not find serializer for %s in mapping' % index_name)
return serializer_class
def get_model_object_by_instance(self, instance):
"""
Provide Model object by elasticsearch response instance.
"""
document = None
_object = None
index_or_alias_name = ElasticsearchUtils.get_alias_by_index_name(
instance.meta.index)
for doc in registry.get_documents():
if index_or_alias_name == doc._index._name: # pylint: disable=protected-access
document = doc
break
hit = self._build_hit(instance)
es_pk = hit['_source'].get('pk')
if document and es_pk:
try:
_object = document(hit).get_queryset().get(pk=es_pk)
except ObjectDoesNotExist:
log.error(
"Object could not be found in database for SearchResult '%r'.",
self)
return _object
def reset_index(self):
""" Deletes and re-creates the Elasticsearch index. """
self.delete_index(self.index)
ElasticsearchUtils.create_alias_and_index(self.es, self.index)
def reset_index(self):
""" Deletes and re-creates the Elasticsearch index. """
self.delete_index(self.index)
ElasticsearchUtils.create_alias_and_index(self.es, self.index)
