def _delete(self, models, options):
index_names = [index._name for index in registry.get_indices(models)]
if not options["force"]:
response = input(
"Are you sure you want to delete "
"the '{}' indexes? [y/N]: ".format(", ".join(index_names))
)
if response.lower() != "y":
self.stdout.write("Aborted")
return False
for index in registry.get_indices(models):
doc = index._doc_types[0]
if getattr(doc, "time_based", None):
self.stdout.write(
"Deletion of time-based indices is not supported yet,"
"delete the index '{}' directly using the Elasticseach deletion API".format(
index._name
)
)
else:
self.stdout.write("Deleting index '{}'".format(index._name))
index.delete(ignore=404)
return True
def test_handle(self, elasticsearch_dsl_default_connection):
""" Verify the command removes all but the newest indexes. """
# Use now as initial time, so indexes are created AFTER the current index so expected values are accurate
initial_time = datetime.datetime.now()
# Create 2 more indexes than we expect to exist after removal
for number in range(
1, settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + 2):
current_time = initial_time + datetime.timedelta(seconds=number)
freezer = freeze_time(current_time)
freezer.start()
for index in registry.get_indices():
ElasticsearchUtils.create_index(index)
freezer.stop()
# Prune indexes and confirm the right indexes are removed
call_command('remove_unused_indexes')
indices_client = elasticsearch_dsl_default_connection.indices
for index in registry.get_indices():
# pylint: disable=protected-access
index_alias = ElasticsearchUtils.get_alias_by_index_name(
index._name)
current_alias = indices_client.get_alias(name=index_alias)
indexes_to_keep = current_alias.keys()
# check that we keep the current indexes, which we don't want removed
all_indexes = self.get_current_index_names(
indices_client=indices_client, index_prefix=index_alias)
assert set(all_indexes).issuperset(set(indexes_to_keep))
# check that other indexes are removed, excepting those that don't hit the retention limit
expected_count = settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + len(
indexes_to_keep)
assert len(all_indexes) == expected_count
# Attempt to prune indexes again and confirm that no indexes are removed
call_command('remove_unused_indexes')
for index in registry.get_indices():
# check that we keep the current indexes, which we don't want removed
# pylint: disable=protected-access
index_alias = ElasticsearchUtils.get_alias_by_index_name(
index._name)
all_indexes = self.get_current_index_names(
indices_client=indices_client, index_prefix=index_alias)
current_alias = indices_client.get_alias(name=index_alias)
indexes_to_keep = current_alias.keys()
assert set(all_indexes).issuperset(set(indexes_to_keep))
# check that index count remains the same as before
expected_count = settings.ELASTICSEARCH_DSL_INDEX_RETENTION_LIMIT + len(
indexes_to_keep)
assert len(all_indexes) == expected_count
# Cleanup indexes created by this test
for index in registry.get_indices():
# pylint: disable=protected-access
indices_client.delete(index=index._name + '_*')
def _delete(self, models, options):
index_names = [str(index) for index in registry.get_indices(models)]
if not options['force']:
response = input("Are you sure you want to delete "
"the '{}' indexes? [n/Y]: ".format(
", ".join(index_names)))
if response.lower() != 'y':
self.stdout.write('Aborted')
return False
for index in registry.get_indices(models):
self.stdout.write("Deleting index '{}'".format(index))
index.delete(ignore=404)
return True
def run_tests(self, test_labels, extra_tests=None, **kwargs):
logging.disable(logging.CRITICAL)
test_settings = {
'AWS_STORAGE_BUCKET_NAME': TEST_BUCKET_NAME,
'ELASTICSEARCH_DSL_AUTOSYNC': False,
}
test_settings.update(get_redis_test_settings())
with override_settings(**test_settings):
redis_client = get_redis_connection()
# prepare test bucket
bucket = default_storage.bucket
bucket.objects.delete()
# create test indices
for index in registry.get_indices():
index._name = f'test-{index._name}'
try:
call_command('search_index', '--create', '-f')
except RequestError:
call_command('search_index', '--delete', '-f')
call_command('search_index', '--create', '-f')
# run tests and make cleanup
try:
return super().run_tests(test_labels, extra_tests, **kwargs)
finally:
redis_client.flushdb()
bucket.objects.delete()
call_command('search_index', '--delete', '-f')
def handle_rebuild(self, models, options, connection):
"""
Rebuild Elasticsearch indices.
Args:
models: An iterable with model classes.
options: The command line options.
connection: The Elasticsearch connection.
"""
index_names = [str(index) for index in registry.get_indices(models)]
if options['current']:
raise CommandError('Using the --current option is not supported with rebuild.')
if not options['force']:
answer = input("Are you sure you want to rebuild indices '{}'? [n/Y]: ".format(", ".join(index_names)))
if answer.lower() != 'y':
self.stdout.write('Aborted')
return False
self.handle_create(models, options, connection)
self.handle_populate(models, options, connection)
self.handle_list(models, options, connection)
self.handle_autoalias(models, options, connection)
self.handle_cleanup(models, options, connection)
def handle_rebuild(self, models, options, connection):
"""
Rebuild Elasticsearch indices.
Args:
models: An iterable with model classes.
options: The command line options.
connection: The Elasticsearch connection.
"""
index_names = [str(index) for index in registry.get_indices(models)]
if options['current']:
raise CommandError(
'Using the --current option is not supported with rebuild.')
if not options['force']:
answer = input(
"Are you sure you want to rebuild indices '{}'? [n/Y]: ".
format(", ".join(index_names)))
if answer.lower() != 'y':
self.stdout.write('Aborted')
return False
self.handle_create(models, options, connection)
self.handle_populate(models, options, connection)
self.handle_list(models, options, connection)
self.handle_autoalias(models, options, connection)
self.handle_cleanup(models, options, connection)
def handle_cleanup(self, models, options, connection):
"""
Delete any inactive indices for the given models.
Args:
models: The models to show the indices for.
options: The command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
indices = connection.indices.get('{}.*'.format(index))
if indices:
old_indices = [
name for name, body in indices.items()
if str(index) not in body.get('aliases', {})
]
if old_indices:
self.stdout.write("Deleting old indices: {}".format(
', '.join(old_indices)))
connection.indices.delete(index=','.join(old_indices))
else:
self.stdout.write(
"No old indices found for {}".format(index))
else:
self.stdout.write('No indices for \'{}\''.format(index))
def handle(self, *args, **options):
self.stdout.write("# Running migrations")
call_command("migrate")
self.stdout.write("# Site settings")
site = Site.objects.get_current()
site.name = settings.SITE_NAME
site.domain = settings.REAL_HOST
site.save()
Site.objects.clear_cache()
if settings.ELASTICSEARCH_ENABLED:
self.stdout.write("# Creating elasticsearch indices")
# The logic comes from django_elasticsearch_dsl.managment.commands.search_index:_create
for index in registry.get_indices(registry.get_models()):
# noinspection PyProtectedMember
self.stdout.write(
f"Creating elasticsearch index '{index._name}' if not exists"
)
# https://elasticsearch-py.readthedocs.io/en/master/api.html:
# "ignore 400 cause by IndexAlreadyExistsException when creating an index"
# See also https://github.com/elastic/elasticsearch/issues/19862
index.create(ignore=400)
else:
self.stdout.write(
"# Elasticsearch is disabled; Not creating any indices")
# This is more brittle, so we run it last
self.stdout.write("# Creating minio buckets")
setup_minio()
logger.info("Setup successful")
def test_create_index(self):
""" Verify the app sets the alias and creates a new index. """
for index in registry.get_indices():
# pylint: disable=protected-access
index_alias = index.get_alias()
index_name, *_ = index_alias.keys()
index._name = index_name
self.es.indices.delete(index=index._name, ignore=404)
assert not self.es.indices.exists(index=index._name)
call_command('install_es_indexes')
# Verify the index was created
for index in registry.get_indices():
# pylint: disable=protected-access
assert self.es.indices.exists(index=index._name)
def refresh_index(self):
"""
Refreshes an index.
https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html
"""
for index in registry.get_indices():
ElasticsearchUtils.refresh_index(self.es, index._name) # pylint: disable=protected-access
def _delete(self, models, options):
# pylint: disable=protected-access
index_names = [index._name for index in registry.get_indices(models)]
if not options['force']:
response = input("Are you sure you want to delete "
"the '{}' indexes? [y/N]: ".format(
", ".join(index_names)))
if response.lower() != 'y':
self.stdout.write('Aborted')
return False
for index in registry.get_indices(models):
try:
index_alias = index.get_alias()
except NotFoundError:
continue
index_name, *_ = index_alias.keys()
index._name = index_name
self.stdout.write("Deleting index '{}'".format(index._name))
index.delete(ignore=404)
return True
def test_alias_exists(self):
""" Verify the app does not setup a new Elasticsearch index if the alias is already set. """
# pylint: disable=protected-access
index_names = [index._name for index in registry.get_indices()]
for index_name in index_names:
# Verify the index exists
assert self.es.indices.exists(index=index_name)
call_command('install_es_indexes')
for index_name in index_names:
# Verify the index still exists
assert self.es.indices.exists(index=index_name)
def handle(self, *args, **options):
for backend_name, host_cong in settings.ELASTICSEARCH_DSL.items():
logger.info(
'Attempting to establish initial connection to Elasticsearch host [%s]...',
host_cong['hosts'])
es_connection = get_connection(backend_name)
es_connection.ping()
logger.info('...success!')
for index in registry.get_indices():
ElasticsearchUtils.create_alias_and_index(
es_connection, index, backend_name)
def _create(self, models, options):
for backend in self.backends:
for index in registry.get_indices(models):
created_index_info = ElasticsearchUtils.create_index(
index, backend)
es_connection = get_connection(backend)
self.stdout.write(
'Creating index "{1}".\nSet alias "{0}" for index "{1}".'.
format(created_index_info.alias, created_index_info.name))
ElasticsearchUtils.set_alias(es_connection,
created_index_info.alias,
created_index_info.name)
def test_index_deletion():
"""Check that deleted persons get deleted from the elasticsearch index"""
for index in registry.get_indices(registry.get_models()):
index.delete(ignore=404)
for index in registry.get_indices(registry.get_models()):
index.create()
old_persons = [
Person(name="Frank Underwood", party="Democrats"),
Person(name="Claire Underwood", party="Democrats"),
]
new_persons = [Person(name="Claire Underwood", party="Democrats")]
old = RisData(sample_city, None, old_persons, [], [], [], [], [], [], 2)
new = RisData(sample_city, None, new_persons, [], [], [], [], [], [], 2)
body = Body(name=old.meta.name, short_name=old.meta.name, ags=old.meta.ags)
body.save()
import_data(body, old)
assert len(MainappSearch({"query": "Underwood"}).execute().hits) == 2
import_data(body, new)
assert len(MainappSearch({"query": "Underwood"}).execute().hits) == 1
def handle_create(self, models, options, connection):
"""
Create new indices and point the aliases to them.
Args:
models: An iterable with model classes.
options: A dict with command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
index_name = '{}.{}'.format(index, int(time.time()))
self.stdout.write("Creating index '{}'".format(index_name))
connection.indices.create(index=index_name, body=index.to_dict())
def handle_create(self, models, options, connection):
"""
Create new indices and point the aliases to them.
Args:
models: An iterable with model classes.
options: A dict with command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
index_name = '{}.{}'.format(index, int(time.time()))
self.stdout.write("Creating index '{}'".format(index_name))
connection.indices.create(index=index_name, body=index.to_dict())
def handle_delete(self, models, options, connection):
"""
Delete the current indices for the given models.
Args:
models: An iterable with model classes.
options: The command line options.
connection: The Elasticsearch connection.
"""
index_names = [str(index) for index in registry.get_indices(models)]
if not options['force']:
response = input(
"Are you sure you want to delete the '{}' indices? [n/Y]: ".format(", ".join(index_names))
)
if response.lower() != 'y':
self.stdout.write('Aborted.')
return False
for index in registry.get_indices(models):
self.stdout.write("Deleting index '{}'".format(index))
connection.indices.delete('{}.*'.format(index), ignore=404)
connection.indices.delete(str(index), ignore=404)
def handle_delete(self, models, options, connection):
"""
Delete the current indices for the given models.
Args:
models: An iterable with model classes.
options: The command line options.
connection: The Elasticsearch connection.
"""
index_names = [str(index) for index in registry.get_indices(models)]
if not options['force']:
response = input(
"Are you sure you want to delete the '{}' indices? [n/Y]: ".
format(", ".join(index_names)))
if response.lower() != 'y':
self.stdout.write('Aborted.')
return False
for index in registry.get_indices(models):
self.stdout.write("Deleting index '{}'".format(index))
connection.indices.delete('{}.*'.format(index), ignore=404)
connection.indices.delete(str(index), ignore=404)
def _create(self, models, options):
for index in registry.get_indices(models):
doc = index._doc_types[0]
if getattr(doc, "time_based", None):
self._check_pipeline_ilm()
mapping = doc._doc_type.mapping.to_dict()
self._check_index_settings(index)
self._check_index_mappings(index, mapping)
self._check_index_template(index)
else:
try:
self.stdout.write(f"Creating index '{index._name}'")
index.create()
except RequestError:
self.stdout.write(f"Index '{index._name}' already exists")
def handle_autoalias(self, models, options, connection):
"""
Update the aliases to point to the latest index.
Args:
models: An iterable with model classes.
options: A dict with command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
indices = connection.indices.get('{}.*'.format(index))
index_names = indices.keys()
index_names.sort()
last_index = index_names[-1]
self.stdout.write("Pointing alias '{}' to '{}'".format(last_index, index))
connection.indices.delete_alias(name=index, index='*', ignore=404)
connection.indices.put_alias(index=last_index, name=index)
def switch_es_index(app_label, model_name, index_name, new_index_name):
model = apps.get_model(app_label, model_name)
indices = registry.get_indices(models=[model])
old_index = _get_index(indices=indices, index_name=index_name)
new_index = old_index.clone(name=new_index_name)
old_index_actual_name = None
if old_index.exists():
# Alias can not be used to delete an index.
# https://www.elastic.co/guide/en/elasticsearch/reference/6.0/indices-delete-index.html
# So get the index actual name to delete it
old_index_info = old_index.get()
# The info is a dictionary and the key is the actual name of the index
old_index_actual_name = list(old_index_info.keys())[0]
# Put alias into the new index name and delete the old index if its exist
new_index.put_alias(name=index_name)
if old_index_actual_name:
old_index.connection.indices.delete(index=old_index_actual_name)
def switch_es_index(app_label, model_name, index_name, new_index_name):
model = apps.get_model(app_label, model_name)
indices = registry.get_indices(models=[model])
old_index = _get_index(indices=indices, index_name=index_name)
new_index = old_index.clone(name=new_index_name)
old_index_actual_name = None
if old_index.exists():
# Alias can not be used to delete an index.
# https://www.elastic.co/guide/en/elasticsearch/reference/6.0/indices-delete-index.html
# So get the index actual name to delete it
old_index_info = old_index.get()
# The info is a dictionary and the key is the actual name of the index
old_index_actual_name = list(old_index_info.keys())[0]
# Put alias into the new index name and delete the old index if its exist
new_index.put_alias(name=index_name)
if old_index_actual_name:
old_index.connection.indices.delete(index=old_index_actual_name)
def handle_autoalias(self, models, options, connection):
"""
Update the aliases to point to the latest index.
Args:
models: An iterable with model classes.
options: A dict with command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
indices = connection.indices.get('{}.*'.format(index))
index_names = indices.keys()
index_names.sort()
last_index = index_names[-1]
self.stdout.write("Pointing alias '{}' to '{}'".format(
last_index, index))
connection.indices.delete_alias(name=index, index='*', ignore=404)
connection.indices.put_alias(index=last_index, name=index)
def pytest_runtest_setup(item):
from django_elasticsearch_dsl.registries import registry
from mcod.core.api.rdf.registry import registry as rdf_registry
import random
import string
es_marker = item.get_closest_marker('elasticsearch')
if es_marker:
for index in registry.get_indices():
index.delete(ignore=404)
index.settings(**settings.ELASTICSEARCH_DSL_INDEX_SETTINGS)
index.create()
sparql_marker = item.get_closest_marker('sparql')
if sparql_marker:
chars = string.ascii_uppercase + string.ascii_lowercase
graph_name = ''.join(random.choice(chars) for _ in range(18))
graph_uri = f'<http://test.mcod/{graph_name}>'
rdf_registry.create_named_graph(graph_uri)
def handle(self, *args, **options):
self.stdout.write("Running migrations")
call_command("migrate")
self.stdout.write("Creating minio buckets")
setup_minio()
if settings.ELASTICSEARCH_ENABLED:
self.stdout.write("Creating elasticsearch indices")
# The logic comes from django_elasticsearch_dsl.managment.commands.search_index:_create
for index in registry.get_indices(registry.get_models()):
self.stdout.write(
"Creating elasticsearch index '{}' if not exists".format(
index._name
)
)
# https://elasticsearch-py.readthedocs.io/en/master/api.html:
# "ignore 400 cause by IndexAlreadyExistsException when creating an index"
# See also https://github.com/elastic/elasticsearch/issues/19862
index.create(ignore=400)
else:
self.stdout.write("Elasticsearch is disabled; Not creating any indices")
def handle_cleanup(self, models, options, connection):
"""
Delete any inactive indices for the given models.
Args:
models: The models to show the indices for.
options: The command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
indices = connection.indices.get('{}.*'.format(index))
if indices:
old_indices = [name for name, body in indices.items() if str(index) not in body.get('aliases', {})]
if old_indices:
self.stdout.write("Deleting old indices: {}".format(', '.join(old_indices)))
connection.indices.delete(index=','.join(old_indices))
else:
self.stdout.write("No old indices found for {}".format(index))
else:
self.stdout.write('No indices for \'{}\''.format(index))
def handle_list(self, models, options, connection):
"""
List the current Elasticsearch indices.
Args:
models: The models to show the indices for.
options: The command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
indices = connection.indices.get('{}.*'.format(index))
if indices:
active_index = None
for index_name, index_body in indices.items():
if str(index) in index_body.get('aliases', {}):
active_index = index_name
break
self.stdout.write('Indices for \'{}\' (current {}): {}'.format(
index, active_index, ', '.join(indices.keys())))
else:
self.stdout.write('No indices for \'{}\''.format(index))
def handle_list(self, models, options, connection):
"""
List the current Elasticsearch indices.
Args:
models: The models to show the indices for.
options: The command line options.
connection: The Elasticsearch connection.
"""
for index in registry.get_indices(models):
indices = connection.indices.get('{}.*'.format(index))
if indices:
active_index = None
for index_name, index_body in indices.items():
if str(index) in index_body.get('aliases', {}):
active_index = index_name
break
self.stdout.write('Indices for \'{}\' (current {}): {}'.format(
index, active_index, ', '.join(indices.keys())
))
else:
self.stdout.write('No indices for \'{}\''.format(index))
def pytest_runtest_teardown(item, nextitem):
from mcod.core.api.rdf.registry import registry as rdf_registry
from django_elasticsearch_dsl.registries import registry
from mcod.resources.indexed_data import es_connections
import shutil
worker = os.environ.get('PYTEST_XDIST_WORKER', '')
archives_path = os.path.join(settings.DATASETS_MEDIA_ROOT, 'archives', worker)
if os.path.exists(archives_path):
shutil.rmtree(archives_path)
es_marker = item.get_closest_marker('elasticsearch')
if es_marker:
worker = os.environ.get('PYTEST_XDIST_WORKER', '')
idx_prefix = getattr(settings, 'ELASTICSEARCH_INDEX_PREFIX', None)
es = es_connections.get_connection()
for res_index in es.indices.get_alias(f'{idx_prefix}-{worker}-*'):
es.indices.delete(index=res_index, ignore=[404])
for index in registry.get_indices():
index.delete(ignore=404)
sparql_marker = item.get_closest_marker('sparql')
if sparql_marker:
rdf_registry.delete_named_graph()
def setup_method(self):
models = registry.get_models()
for index in registry.get_indices(models):
index.delete(ignore=404)
index.create()
def create_new_es_index(app_label, model_name, index_name, new_index_name):
model = apps.get_model(app_label, model_name)
indices = registry.get_indices(models=[model])
old_index = _get_index(indices=indices, index_name=index_name)
new_index = old_index.clone(name=new_index_name)
new_index.create()
def _delete(self, models):
for index in registry.get_indices(models):
index.delete(ignore=404)
return True
def _create(self, models):
for index in registry.get_indices(models):
index.create()
def _create(self, models, options):
for index in registry.get_indices(models):
self.stdout.write("Creating index '{}'".format(index))
index.create()
def create_new_es_index(app_label, model_name, index_name, new_index_name):
model = apps.get_model(app_label, model_name)
indices = registry.get_indices(models=[model])
old_index = _get_index(indices=indices, index_name=index_name)
new_index = old_index.clone(name=new_index_name)
new_index.create()