def index(self, index_name, specific_targets=None, ids=None):
"""
Index objects from our index-enabled models.
"""
for mapping_class in ModelMappings.get_model_mappings().values():
model = mapping_class.get_model()
# Skip this model if there are specific targets and not specified.
if not self.model_targetted(model, specific_targets):
continue
model = mapping_class.get_model()
if ids:
for arg_id in ids:
try:
obj = model.objects.get(id=arg_id)
except model.DoesNotExist:
self.stdout.write(
'ID does not exist, deleting %s for %s' %
(arg_id, self.full_name(model)))
mapping_class.unindex(arg_id, index=index_name)
else:
if mapping_class.has_deleted() and obj.is_deleted:
self.stdout.write(
'ID is_deleted, deleting %s for %s' %
(arg_id, self.full_name(model)))
mapping_class.unindex(arg_id, index=index_name)
else:
self.stdout.write('ID updating %s for %s' %
(arg_id, self.full_name(model)))
document = mapping_class.extract_document(
arg_id, obj)
mapping_class.bulk_index([document],
id_field='id',
index=index_name)
return
self.stdout.write('Indexing %s' % self.full_name(model))
if mapping_class.has_deleted():
model_objs = model.objects.filter(is_deleted=False)
else:
model_objs = model.objects.all()
index_objects(mapping_class,
model_objs,
index_name,
print_progress=True)
def index(self, index_name, specific_targets=None, ids=None):
"""
Index objects from our index-enabled models.
"""
for mapping_class in ModelMappings.get_model_mappings().values():
model = mapping_class.get_model()
# Skip this model if there are specific targets and not specified.
if not self.model_targetted(model, specific_targets):
continue
model = mapping_class.get_model()
if ids:
for arg_id in ids:
try:
obj = model.objects.get(id=arg_id)
except model.DoesNotExist:
self.stdout.write('ID does not exist, deleting %s for %s' % (arg_id, self.full_name(model)))
mapping_class.unindex(arg_id, index=index_name)
else:
if mapping_class.has_deleted() and obj.is_deleted:
self.stdout.write('ID is_deleted, deleting %s for %s' % (arg_id, self.full_name(model)))
mapping_class.unindex(arg_id, index=index_name)
else:
self.stdout.write('ID updating %s for %s' % (arg_id, self.full_name(model)))
document = mapping_class.extract_document(arg_id, obj)
mapping_class.bulk_index([document], id_field='id', index=index_name)
return
self.stdout.write('Indexing %s' % self.full_name(model))
if mapping_class.has_deleted():
model_objs = model.objects.filter(is_deleted=False)
else:
model_objs = model.objects.all()
index_objects(mapping_class, model_objs, index_name, print_progress=True)
def scan_indexes(self, local_apps):
"""
Scan the installed apps for indexes.
"""
ModelMappings.scan(local_apps)
def handle(self, *args, **options):
es = get_es_client()
if args:
self.stdout.write('Aborting, unexpected arguments %s' % list(args))
return
if options['list']:
self.stdout.write('Possible models to index:\n')
for mapping in ModelMappings.get_model_mappings().values():
self.stdout.write(mapping.get_mapping_type_name())
return
target = options['target']
if target:
targets = target.split(',')
else:
targets = [] # (meaning all)
has_targets = targets != []
self.stdout.write('Please remember that HelloLily needs to be in maintenance mode. \n\n')
if has_targets:
# Do a quick run to check if all targets are valid models.
check_targets = list(targets) # make a copy
for target in check_targets:
for mapping in ModelMappings.get_model_mappings().values():
if self.model_targetted(mapping, [target]):
check_targets.remove(target)
break
if check_targets:
self.stdout.write('Aborting, following targets not recognized: %s' % check_targets)
return
for mapping in ModelMappings.get_model_mappings().values():
model_name = mapping.get_mapping_type_name()
main_index_base = settings.ES_INDEXES['default']
main_index = get_index_name(main_index_base, mapping)
# Skip this model if there are specific targets and not specified.
if has_targets and not self.model_targetted(mapping, targets):
continue
self.stdout.write('==> %s' % model_name)
# Check if we currently have an index for this mapping.
old_index = None
aliases = es.indices.get_aliases(name=main_index)
for key, value in aliases.iteritems():
if value['aliases']:
old_index = key
self.stdout.write('Current index "%s"' % key)
# Check any indices with no alias (leftovers from failed indexing).
# Or it could be that it is still in progress,
aliases = es.indices.get_aliases()
for key, value in aliases.iteritems():
if not key.endswith(model_name):
# Not the model we are looking after.
continue
if key == main_index:
# This is an auto created index. Will be removed at end of command.
continue
if not value['aliases']:
if options['force']:
self.stdout.write('Removing leftover "%s"' % key)
es.indices.delete(key)
else:
raise Exception('Found leftover %s, proceed with -f to remove.'
' Make sure indexing this model is not already running!' % key)
# Create new index.
index_settings = {
'mappings': {
model_name: mapping.get_mapping()
},
'settings': {
'analysis': get_analyzers()['analysis'],
'number_of_shards': 1,
}
}
temp_index_base = 'index_%s' % (int(time.time()))
temp_index = get_index_name(temp_index_base, mapping)
self.stdout.write('Creating new index "%s"' % temp_index)
es.indices.create(temp_index, body=index_settings)
# Index documents.
self.index_documents(mapping, temp_index_base)
# Switch aliases.
if old_index:
es.indices.update_aliases({
'actions': [
{'remove': {'index': old_index, 'alias': main_index}},
{'remove': {'index': old_index, 'alias': main_index_base}},
{'add': {'index': temp_index, 'alias': main_index}},
{'add': {'index': temp_index, 'alias': main_index_base}},
]
})
self.stdout.write('Removing previous index "%s"' % old_index)
es.indices.delete(old_index)
else:
if es.indices.exists(main_index):
# This is a corner case. There was no alias named index_name, but
# an index index_name nevertheless exists, this only happens when the index
# was already created (because of ES auto creation features).
self.stdout.write('Removing previous (presumably auto created) index "%s"' % main_index)
es.indices.delete(main_index)
es.indices.update_aliases({
'actions': [
{'add': {'index': temp_index, 'alias': main_index}},
{'add': {'index': temp_index, 'alias': main_index_base}},
]
})
self.stdout.write('')
self.stdout.write('Indexing finished.')
if options['queries']:
from django.db import connection
for query in connection.queries:
print query
def handle(self, *args, **options):
es = get_es_client()
# Check if specific targets specified to run, or otherwise run all.
target = options['target']
if target:
targets = target.split(',')
# Check specific IDs.
if options['id']:
ids = options['id'].split(',')
self.index(settings.ES_INDEXES['default'], specific_targets=targets, ids=ids)
else:
self.index(settings.ES_INDEXES['default'], specific_targets=targets)
else:
if es.indices.exists(settings.ES_INDEXES['new_index']):
if options['force']:
es.indices.delete(settings.ES_INDEXES['new_index'])
else:
raise Exception('%s already exists, run with -f to remove old' % settings.ES_INDEXES['new_index'])
# Define analyzers and set alias for new index.
index_settings = {
'mappings': {},
'settings': get_analyzers(),
'aliases': {
settings.ES_INDEXES['new_index']: {},
},
}
# Retrieve the mappings for the index-enabled models.
for mapping_class in ModelMappings.get_model_mappings().values():
model_name = mapping_class.get_mapping_type_name()
index_settings['mappings'].update({model_name: mapping_class.get_mapping()})
new_index = 'index_%s' % (int(time.time()))
self.stdout.write('Creating new index "%s"' % new_index)
es.indices.create(new_index, body=index_settings)
self.index(new_index)
# The default index name, (we will use as an alias).
index_name = settings.ES_INDEXES['default']
# Check if we have a current index.
old_index = None
aliases = es.indices.get_aliases(name=index_name)
for key, value in aliases.iteritems():
if value['aliases']:
old_index = key
# Change the alias to point to our new index, and remove the old index.
self.stdout.write('Changing alias "%s" from old index "%s" to new index "%s"' %
(index_name, old_index, new_index))
if old_index:
es.indices.update_aliases({
'actions': [
{'remove': {'index': old_index, 'alias': index_name}},
{'add': {'index': new_index, 'alias': index_name}},
]
})
es.indices.delete(old_index)
else:
if es.indices.exists(index_name):
# This is a corner case. There was no alias named index_name, but
# an index index_name nevertheless exists, this only happens when the index
# was already created (because of ES auto creation features).
es.indices.delete(index_name)
es.indices.update_aliases({
'actions': [
{'add': {'index': new_index, 'alias': index_name}}
]
})
es.indices.update_aliases({
'actions': [
{'remove': {'index': new_index, 'alias': settings.ES_INDEXES['new_index']}},
]
})
# Compensate for the small race condition in the signal handlers:
# They check if the index exists then updates documents, however when we
# delete the alias in between these two commands, we can end up with
# an auto created index named new_index, so we simply delete it again.
time.sleep(5)
try:
es.indices.delete(settings.ES_INDEXES['new_index'])
except NotFoundError:
pass
if options['queries']:
from django.db import connection
for query in connection.queries:
print query
def handle(self, *args, **options):
es = get_es_client()
# Check if specific targets specified to run, or otherwise run all.
target = options['target']
if target:
targets = target.split(',')
# Check specific IDs.
if options['id']:
ids = options['id'].split(',')
self.index(settings.ES_INDEXES['default'],
specific_targets=targets,
ids=ids)
else:
self.index(settings.ES_INDEXES['default'],
specific_targets=targets)
else:
if es.indices.exists(settings.ES_INDEXES['new_index']):
if options['force']:
es.indices.delete(settings.ES_INDEXES['new_index'])
else:
raise Exception(
'%s already exists, run with -f to remove old' %
settings.ES_INDEXES['new_index'])
# Define analyzers and set alias for new index.
index_settings = {
'mappings': {},
'settings': get_analyzers(),
'aliases': {
settings.ES_INDEXES['new_index']: {},
},
}
# Retrieve the mappings for the index-enabled models.
for mapping_class in ModelMappings.get_model_mappings().values():
model_name = mapping_class.get_mapping_type_name()
index_settings['mappings'].update(
{model_name: mapping_class.get_mapping()})
new_index = 'index_%s' % (int(time.time()))
self.stdout.write('Creating new index "%s"' % new_index)
es.indices.create(new_index, body=index_settings)
self.index(new_index)
# The default index name, (we will use as an alias).
index_name = settings.ES_INDEXES['default']
# Check if we have a current index.
old_index = None
aliases = es.indices.get_aliases(name=index_name)
for key, value in aliases.iteritems():
if value['aliases']:
old_index = key
# Change the alias to point to our new index, and remove the old index.
self.stdout.write(
'Changing alias "%s" from old index "%s" to new index "%s"' %
(index_name, old_index, new_index))
if old_index:
es.indices.update_aliases({
'actions': [
{
'remove': {
'index': old_index,
'alias': index_name
}
},
{
'add': {
'index': new_index,
'alias': index_name
}
},
]
})
es.indices.delete(old_index)
else:
if es.indices.exists(index_name):
# This is a corner case. There was no alias named index_name, but
# an index index_name nevertheless exists, this only happens when the index
# was already created (because of ES auto creation features).
es.indices.delete(index_name)
es.indices.update_aliases({
'actions': [{
'add': {
'index': new_index,
'alias': index_name
}
}]
})
es.indices.update_aliases({
'actions': [
{
'remove': {
'index': new_index,
'alias': settings.ES_INDEXES['new_index']
}
},
]
})
# Compensate for the small race condition in the signal handlers:
# They check if the index exists then updates documents, however when we
# delete the alias in between these two commands, we can end up with
# an auto created index named new_index, so we simply delete it again.
time.sleep(5)
try:
es.indices.delete(settings.ES_INDEXES['new_index'])
except NotFoundError:
pass
if options['queries']:
from django.db import connection
for query in connection.queries:
print query
def scan_indexes(self, local_apps):
"""
Scan the installed apps for indexes.
"""
ModelMappings.scan(local_apps)