def refresh_index(self): """ Refreshes an index. https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html """ ElasticsearchUtils.refresh_index(self.es, self.index)
def refresh_index(self): """ Refreshes an index. https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html """ for index in registry.get_indices(): ElasticsearchUtils.refresh_index(self.es, index._name) # pylint: disable=protected-access
def test_handle(self, elasticsearch_dsl_default_connection): """ Verify the command removes all but the newest indexes. """ # Use now as initial time, so indexes are created AFTER the current index so expected values are accurate initial_time = datetime.datetime.now() # Create 2 more indexes than we expect to exist after removal for number in range( 1, settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + 2): current_time = initial_time + datetime.timedelta(seconds=number) freezer = freeze_time(current_time) freezer.start() for index in registry.get_indices(): ElasticsearchUtils.create_index(index) freezer.stop() # Prune indexes and confirm the right indexes are removed call_command('remove_unused_indexes') indices_client = elasticsearch_dsl_default_connection.indices for index in registry.get_indices(): # pylint: disable=protected-access index_alias = ElasticsearchUtils.get_alias_by_index_name( index._name) current_alias = indices_client.get_alias(name=index_alias) indexes_to_keep = current_alias.keys() # check that we keep the current indexes, which we don't want removed all_indexes = self.get_current_index_names( indices_client=indices_client, index_prefix=index_alias) assert set(all_indexes).issuperset(set(indexes_to_keep)) # check that other indexes are removed, excepting those that don't hit the retention limit expected_count = settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + len( indexes_to_keep) assert len(all_indexes) == expected_count # Attempt to prune indexes again and confirm that no indexes are removed call_command('remove_unused_indexes') for index in registry.get_indices(): # check that we keep the current indexes, which we don't want removed # pylint: disable=protected-access index_alias = ElasticsearchUtils.get_alias_by_index_name( index._name) all_indexes = self.get_current_index_names( indices_client=indices_client, index_prefix=index_alias) current_alias = indices_client.get_alias(name=index_alias) indexes_to_keep = current_alias.keys() assert set(all_indexes).issuperset(set(indexes_to_keep)) # check that index count remains the same as before expected_count = settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + len( indexes_to_keep) assert len(all_indexes) == expected_count # Cleanup indexes created by this test for index in registry.get_indices(): # pylint: disable=protected-access indices_client.delete(index=index._name + '_*')
def handle(self, *args, **options): host = settings.HAYSTACK_CONNECTIONS['default']['URL'] alias = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'] logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host) es = Elasticsearch(host) logger.info('...success!') ElasticsearchUtils.create_alias_and_index(es, alias)
def handle(self, *args, **options): host = settings.HAYSTACK_CONNECTIONS['default']['URL'] alias = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'] logger.info( 'Attempting to establish initial connection to Elasticsearch host [%s]...', host) es = Elasticsearch(host) logger.info('...success!') ElasticsearchUtils.create_alias_and_index(es, alias)
def _update(self, models, options): """ Update indices with sanity check. Will be created a new index and populate with data. The index will be masked with previous one to prevent missing data. """ alias_mappings = [] for document in registry.get_documents(models): # pylint: disable=protected-access index = document._index record_count = self.get_record_count(document) alias, new_index_name = self.prepare_backend_index(index) alias_mappings.append( AliasMapper(document, index, new_index_name, alias, record_count)) # Set the alias (from settings) to the timestamped catalog. run_attempts = 0 indexes_pending = { key: '' for key in [x.new_index_name for x in alias_mappings] } conn = get_connection() while indexes_pending and run_attempts < 1: # Only try once, as retries gave buggy results. See VAN-391 run_attempts += 1 self._populate(models, options) for doc, __, new_index_name, alias, record_count in alias_mappings: # Run a sanity check to ensure we aren't drastically changing the # index, which could be indicative of a bug. if new_index_name in indexes_pending and not options.get( 'disable_change_limit', False): record_count_is_sane, index_info_string = self.sanity_check_new_index( run_attempts, doc, new_index_name, record_count) if record_count_is_sane: ElasticsearchUtils.set_alias(conn, alias, new_index_name) indexes_pending.pop(new_index_name, None) else: indexes_pending[new_index_name] = index_info_string else: ElasticsearchUtils.set_alias(conn, alias, new_index_name) indexes_pending.pop(new_index_name, None) for index_alias_mapper in alias_mappings: index_alias_mapper.registered_index._name = index_alias_mapper.alias # pylint: disable=protected-access if indexes_pending: raise CommandError( 'Sanity check failed for the new index(es): {}'.format( indexes_pending)) return True
def handle(self, *args, **options): for backend_name, host_cong in settings.ELASTICSEARCH_DSL.items(): logger.info( 'Attempting to establish initial connection to Elasticsearch host [%s]...', host_cong['hosts']) es_connection = get_connection(backend_name) es_connection.ping() logger.info('...success!') for index in registry.get_indices(): ElasticsearchUtils.create_alias_and_index( es_connection, index, backend_name)
def _create(self, models, options): for backend in self.backends: for index in registry.get_indices(models): created_index_info = ElasticsearchUtils.create_index( index, backend) es_connection = get_connection(backend) self.stdout.write( 'Creating index "{1}".\nSet alias "{0}" for index "{1}".'. format(created_index_info.alias, created_index_info.name)) ElasticsearchUtils.set_alias(es_connection, created_index_info.alias, created_index_info.name)
def prepare_backend_index(self, registered_index, backend='default'): """ Prepares an index that will be used to store data by the backend. Args: registered_index (Index): django elasticsearch index instance. backend (ElasticsearchSearchBackend): Backend to update. Returns: (tuple): tuple containing: alias(str): Recommended alias for the new index. index_name(str): Name of the newly-created index. """ timestamp = datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S') # pylint: disable=protected-access computed_alias = ElasticsearchUtils.get_alias_by_index_name( registered_index._name) new_index_name = '{alias}_{timestamp}'.format(alias=computed_alias, timestamp=timestamp) registered_index._name = new_index_name self.stdout.write("Creating index '{}'".format(registered_index._name)) registered_index.create(using=backend) return computed_alias, new_index_name
def test_handle(self): """ Verify the command removes all but the newest indexes. """ # Create initial index with alias ElasticsearchUtils.create_alias_and_index( es_connection=self.backend.conn, alias=self.backend.index_name) # Use now as initial time, so indexes are created AFTER the current index so expected values are accurate initial_time = datetime.datetime.now() # Create 2 more indexes than we expect to exist after removal for number in range(1, settings.HAYSTACK_INDEX_RETENTION_LIMIT + 2): current_time = initial_time + datetime.timedelta(seconds=number) freezer = freeze_time(current_time) freezer.start() ElasticsearchUtils.create_index(es_connection=self.backend.conn, prefix=self.backend.index_name) freezer.stop() # Prune indexes and confirm the right indexes are removed call_command('remove_unused_indexes') current_alias_name = self.backend.index_name indices_client = self.backend.conn.indices current_alias = indices_client.get_alias(name=current_alias_name) indexes_to_keep = current_alias.keys() # check that we keep the current indexes, which we don't want removed all_indexes = self.get_current_index_names( indices_client=indices_client, index_prefix=self.backend.index_name) assert set(all_indexes).issuperset(set(indexes_to_keep)) # check that other indexes are removed, excepting those that don't hit the retention limit expected_count = settings.HAYSTACK_INDEX_RETENTION_LIMIT + len( indexes_to_keep) assert len(all_indexes) == expected_count # Attempt to prune indexes again and confirm that no indexes are removed call_command('remove_unused_indexes') # check that we keep the current indexes, which we don't want removed all_indexes = self.get_current_index_names( indices_client=indices_client, index_prefix=self.backend.index_name) assert set(all_indexes).issuperset(set(indexes_to_keep)) # check that index count remains the same as before assert len(all_indexes) == expected_count
def get_current_index_names(indices_client, index_prefix): all_indexes = indices_client.get('*').keys() all_current_indexes = [ index_name for index_name in all_indexes if ElasticsearchUtils.get_alias_by_index_name(index_name) == index_prefix ] return all_current_indexes
def haystack_default_connection(haystack_add_xdist_suffix_to_index_name): # pylint: disable=redefined-outer-name,unused-argument skip_if_no_django() backend = haystack_connections['default'].get_backend() # Force Haystack to update the mapping for the index backend.setup_complete = False es = backend.conn index_name = backend.index_name ElasticsearchUtils.delete_index(es, index_name) ElasticsearchUtils.create_alias_and_index(es, index_name) ElasticsearchUtils.refresh_index(es, index_name) yield backend ElasticsearchUtils.delete_index(es, index_name)
def prepare_backend_index(self, backend): """ Prepares an index that will be used to store data by the backend. Args: backend (ElasticsearchSearchBackend): Backend to update. Returns: (tuple): tuple containing: alias(str): Recommended alias for the new index. index_name(str): Name of the newly-created index. """ alias = backend.index_name index_name = ElasticsearchUtils.create_index(backend.conn, alias) backend.index_name = index_name return alias, index_name
def get_serializer_class_by_instance(self, instance): if isinstance(instance, EmptySearch): serializer_class = serializers.Serializer else: instance_serializers = { # pylint: disable=protected-access document._index._name: serializer for document, serializer in self.Meta.serializers.items() } index_name = instance.meta['index'] index_alias = ElasticsearchUtils.get_alias_by_index_name( index_name) serializer_class = instance_serializers.get(index_alias, None) if not serializer_class: raise ImproperlyConfigured( 'Could not find serializer for %s in mapping' % index_name) return serializer_class
def get_model_object_by_instance(self, instance): """ Provide Model object by elasticsearch response instance. """ document = None _object = None index_or_alias_name = ElasticsearchUtils.get_alias_by_index_name( instance.meta.index) for doc in registry.get_documents(): if index_or_alias_name == doc._index._name: # pylint: disable=protected-access document = doc break hit = self._build_hit(instance) es_pk = hit['_source'].get('pk') if document and es_pk: try: _object = document(hit).get_queryset().get(pk=es_pk) except ObjectDoesNotExist: log.error( "Object could not be found in database for SearchResult '%r'.", self) return _object
def reset_index(self): """ Deletes and re-creates the Elasticsearch index. """ self.delete_index(self.index) ElasticsearchUtils.create_alias_and_index(self.es, self.index)