def handle(self, *args, **kwargs):
self.stdout.write('Please remember that Lily needs to be in maintenance mode. \n\n')
try:
self.handle_args(*args)
self.handle_kwargs(**kwargs)
self.es = get_es_client()
self.index()
if self.log_queries:
for query in connection.queries:
print query
except Exception, e:
self.stderr.write('\nIndexing error.\n')
traceback.print_exc()
heroku_env = os.environ.get('HEROKU_ENV')
if heroku_env == 'production':
travis_build = os.environ.get('TRAVIS_BUILD_ID')
travis_link = 'https://travis-ci.org/HelloLily/hellolily/builds/{0}'.format(travis_build)
slack = Slacker(os.environ.get('SLACK_API_TOKEN'))
slack.chat.post_message(
'#lily_ci',
'Indexing failed with reason `{0}` in Travis build {1}.'.format(repr(e), travis_link)
)
def handle(self, *args, **kwargs):
self.stdout.write(
'Please remember that Lily needs to be in maintenance mode. \n\n')
try:
self.handle_args(*args)
self.handle_kwargs(**kwargs)
self.es = get_es_client()
self.index()
if self.log_queries:
for query in connection.queries:
print query
except Exception, e:
self.stderr.write('\nIndexing error.\n')
traceback.print_exc()
heroku_env = os.environ.get('HEROKU_ENV')
if heroku_env == 'production':
travis_build = os.environ.get('TRAVIS_BUILD_ID')
travis_link = 'https://travis-ci.org/HelloLily/hellolily/builds/{0}'.format(
travis_build)
slack = Slacker(os.environ.get('SLACK_API_TOKEN'))
slack.chat.post_message(
'#lily_ci',
'Indexing failed with reason `{0}` in Travis build {1}.'.
format(repr(e), travis_link))
def handle(self, *args, **kwargs):
self.stdout.write('Please remember that Lily needs to be in maintenance mode. \n\n')
try:
self.handle_args(*args)
self.handle_kwargs(**kwargs)
self.es = get_es_client()
self.index()
if self.log_queries:
for query in connection.queries:
print query
except Exception:
self.stderr.write('\nIndexing error.\n')
traceback.print_exc()
def handle(self, *args, **kwargs):
self.stdout.write('Please remember that Lily needs to be in maintenance mode. \n\n')
try:
self.handle_args(*args)
self.handle_kwargs(**kwargs)
self.es = get_es_client()
self.index()
if self.log_queries:
for query in connection.queries:
print query
except Exception:
self.stderr.write('\nIndexing error.\n')
traceback.print_exc()
def handle(self, *args, **options):
es = get_es_client()
cluster = es.nodes.stats()['nodes'].values()[0]['host'] # pretty host name
if '.' not in cluster:
cluster = es.nodes.stats()['nodes'].values()[0]['ip'][0] # use ip instead
self.stdout.write('This command will irreversibly DELETE ALL INDEXES from cluster %s!' % cluster)
confirm = None
while True:
if confirm == 'yes':
self.stdout.write('Proceeding.')
break
elif confirm == 'no':
self.stdout.write('Aborted.')
return
confirm = raw_input('Do you wish to proceed? Enter "yes" or "no": ')
self.stdout.write('Deleting all indexes in 10 seconds. Hit ctrl+c to abort.')
time.sleep(10)
result = es.indices.delete('_all')
self.stdout.write(str(result))
from datetime import date
import logging
import traceback
from django.conf import settings
from elasticsearch.exceptions import NotFoundError
from elasticutils.contrib.django import tasks
from lily.search.connections_utils import get_es_client, get_index_name
from lily.utils import logutil
logger = logging.getLogger('search')
main_index = settings.ES_INDEXES['default']
es = get_es_client(maxsize=1)
def update_in_index(instance, mapping):
"""
Utility function for signal listeners index to Elasticsearch.
Currently uses synchronous tasks. And because of that all exceptions are
caught, so failures will not interfere with the regular model updates.
"""
if settings.ES_DISABLED:
return
if hasattr(instance, 'is_deleted') and instance.is_deleted:
remove_from_index(instance, mapping)
else:
logger.info(u'Updating instance %s: %s' % (instance.__class__.__name__, instance.pk))
try:
from datetime import date
import logging
import traceback
from django.conf import settings
from elasticsearch.exceptions import NotFoundError
from elasticutils.contrib.django import tasks
from lily.search.connections_utils import get_es_client, get_index_name
from lily.utils import logutil
logger = logging.getLogger('search')
main_index = settings.ES_INDEXES['default']
es = get_es_client(maxsize=1)
def update_in_index(instance, mapping):
"""
Utility function for signal listeners index to Elasticsearch.
Currently uses synchronous tasks. And because of that all exceptions are
caught, so failures will not interfere with the regular model updates.
"""
if settings.ES_DISABLED:
return
if hasattr(instance, 'is_deleted') and instance.is_deleted:
remove_from_index(instance, mapping)
else:
logger.info(u'Updating instance %s: %s' %
(instance.__class__.__name__, instance.pk))
try:
def handle(self, *args, **options):
es = get_es_client()
if args:
self.stdout.write('Aborting, unexpected arguments %s' % list(args))
return
if options['list']:
self.stdout.write('Possible models to index:\n')
for mapping in ModelMappings.get_model_mappings().values():
self.stdout.write(mapping.get_mapping_type_name())
return
target = options['target']
if target:
targets = target.split(',')
else:
targets = [] # (meaning all)
has_targets = targets != []
self.stdout.write('Please remember that HelloLily needs to be in maintenance mode. \n\n')
if has_targets:
# Do a quick run to check if all targets are valid models.
check_targets = list(targets) # make a copy
for target in check_targets:
for mapping in ModelMappings.get_model_mappings().values():
if self.model_targetted(mapping, [target]):
check_targets.remove(target)
break
if check_targets:
self.stdout.write('Aborting, following targets not recognized: %s' % check_targets)
return
for mapping in ModelMappings.get_model_mappings().values():
model_name = mapping.get_mapping_type_name()
main_index_base = settings.ES_INDEXES['default']
main_index = get_index_name(main_index_base, mapping)
# Skip this model if there are specific targets and not specified.
if has_targets and not self.model_targetted(mapping, targets):
continue
self.stdout.write('==> %s' % model_name)
# Check if we currently have an index for this mapping.
old_index = None
aliases = es.indices.get_aliases(name=main_index)
for key, value in aliases.iteritems():
if value['aliases']:
old_index = key
self.stdout.write('Current index "%s"' % key)
# Check any indices with no alias (leftovers from failed indexing).
# Or it could be that it is still in progress,
aliases = es.indices.get_aliases()
for key, value in aliases.iteritems():
if not key.endswith(model_name):
# Not the model we are looking after.
continue
if key == main_index:
# This is an auto created index. Will be removed at end of command.
continue
if not value['aliases']:
if options['force']:
self.stdout.write('Removing leftover "%s"' % key)
es.indices.delete(key)
else:
raise Exception('Found leftover %s, proceed with -f to remove.'
' Make sure indexing this model is not already running!' % key)
# Create new index.
index_settings = {
'mappings': {
model_name: mapping.get_mapping()
},
'settings': {
'analysis': get_analyzers()['analysis'],
'number_of_shards': 1,
}
}
temp_index_base = 'index_%s' % (int(time.time()))
temp_index = get_index_name(temp_index_base, mapping)
self.stdout.write('Creating new index "%s"' % temp_index)
es.indices.create(temp_index, body=index_settings)
# Index documents.
self.index_documents(mapping, temp_index_base)
# Switch aliases.
if old_index:
es.indices.update_aliases({
'actions': [
{'remove': {'index': old_index, 'alias': main_index}},
{'remove': {'index': old_index, 'alias': main_index_base}},
{'add': {'index': temp_index, 'alias': main_index}},
{'add': {'index': temp_index, 'alias': main_index_base}},
]
})
self.stdout.write('Removing previous index "%s"' % old_index)
es.indices.delete(old_index)
else:
if es.indices.exists(main_index):
# This is a corner case. There was no alias named index_name, but
# an index index_name nevertheless exists, this only happens when the index
# was already created (because of ES auto creation features).
self.stdout.write('Removing previous (presumably auto created) index "%s"' % main_index)
es.indices.delete(main_index)
es.indices.update_aliases({
'actions': [
{'add': {'index': temp_index, 'alias': main_index}},
{'add': {'index': temp_index, 'alias': main_index_base}},
]
})
self.stdout.write('')
self.stdout.write('Indexing finished.')
if options['queries']:
from django.db import connection
for query in connection.queries:
print query
def handle(self, *args, **options):
es = get_es_client()
# Check if specific targets specified to run, or otherwise run all.
target = options['target']
if target:
targets = target.split(',')
# Check specific IDs.
if options['id']:
ids = options['id'].split(',')
self.index(settings.ES_INDEXES['default'], specific_targets=targets, ids=ids)
else:
self.index(settings.ES_INDEXES['default'], specific_targets=targets)
else:
if es.indices.exists(settings.ES_INDEXES['new_index']):
if options['force']:
es.indices.delete(settings.ES_INDEXES['new_index'])
else:
raise Exception('%s already exists, run with -f to remove old' % settings.ES_INDEXES['new_index'])
# Define analyzers and set alias for new index.
index_settings = {
'mappings': {},
'settings': get_analyzers(),
'aliases': {
settings.ES_INDEXES['new_index']: {},
},
}
# Retrieve the mappings for the index-enabled models.
for mapping_class in ModelMappings.get_model_mappings().values():
model_name = mapping_class.get_mapping_type_name()
index_settings['mappings'].update({model_name: mapping_class.get_mapping()})
new_index = 'index_%s' % (int(time.time()))
self.stdout.write('Creating new index "%s"' % new_index)
es.indices.create(new_index, body=index_settings)
self.index(new_index)
# The default index name, (we will use as an alias).
index_name = settings.ES_INDEXES['default']
# Check if we have a current index.
old_index = None
aliases = es.indices.get_aliases(name=index_name)
for key, value in aliases.iteritems():
if value['aliases']:
old_index = key
# Change the alias to point to our new index, and remove the old index.
self.stdout.write('Changing alias "%s" from old index "%s" to new index "%s"' %
(index_name, old_index, new_index))
if old_index:
es.indices.update_aliases({
'actions': [
{'remove': {'index': old_index, 'alias': index_name}},
{'add': {'index': new_index, 'alias': index_name}},
]
})
es.indices.delete(old_index)
else:
if es.indices.exists(index_name):
# This is a corner case. There was no alias named index_name, but
# an index index_name nevertheless exists, this only happens when the index
# was already created (because of ES auto creation features).
es.indices.delete(index_name)
es.indices.update_aliases({
'actions': [
{'add': {'index': new_index, 'alias': index_name}}
]
})
es.indices.update_aliases({
'actions': [
{'remove': {'index': new_index, 'alias': settings.ES_INDEXES['new_index']}},
]
})
# Compensate for the small race condition in the signal handlers:
# They check if the index exists then updates documents, however when we
# delete the alias in between these two commands, we can end up with
# an auto created index named new_index, so we simply delete it again.
time.sleep(5)
try:
es.indices.delete(settings.ES_INDEXES['new_index'])
except NotFoundError:
pass
if options['queries']:
from django.db import connection
for query in connection.queries:
print query
def handle(self, *args, **options):
es = get_es_client()
# Check if specific targets specified to run, or otherwise run all.
target = options['target']
if target:
targets = target.split(',')
# Check specific IDs.
if options['id']:
ids = options['id'].split(',')
self.index(settings.ES_INDEXES['default'],
specific_targets=targets,
ids=ids)
else:
self.index(settings.ES_INDEXES['default'],
specific_targets=targets)
else:
if es.indices.exists(settings.ES_INDEXES['new_index']):
if options['force']:
es.indices.delete(settings.ES_INDEXES['new_index'])
else:
raise Exception(
'%s already exists, run with -f to remove old' %
settings.ES_INDEXES['new_index'])
# Define analyzers and set alias for new index.
index_settings = {
'mappings': {},
'settings': get_analyzers(),
'aliases': {
settings.ES_INDEXES['new_index']: {},
},
}
# Retrieve the mappings for the index-enabled models.
for mapping_class in ModelMappings.get_model_mappings().values():
model_name = mapping_class.get_mapping_type_name()
index_settings['mappings'].update(
{model_name: mapping_class.get_mapping()})
new_index = 'index_%s' % (int(time.time()))
self.stdout.write('Creating new index "%s"' % new_index)
es.indices.create(new_index, body=index_settings)
self.index(new_index)
# The default index name, (we will use as an alias).
index_name = settings.ES_INDEXES['default']
# Check if we have a current index.
old_index = None
aliases = es.indices.get_aliases(name=index_name)
for key, value in aliases.iteritems():
if value['aliases']:
old_index = key
# Change the alias to point to our new index, and remove the old index.
self.stdout.write(
'Changing alias "%s" from old index "%s" to new index "%s"' %
(index_name, old_index, new_index))
if old_index:
es.indices.update_aliases({
'actions': [
{
'remove': {
'index': old_index,
'alias': index_name
}
},
{
'add': {
'index': new_index,
'alias': index_name
}
},
]
})
es.indices.delete(old_index)
else:
if es.indices.exists(index_name):
# This is a corner case. There was no alias named index_name, but
# an index index_name nevertheless exists, this only happens when the index
# was already created (because of ES auto creation features).
es.indices.delete(index_name)
es.indices.update_aliases({
'actions': [{
'add': {
'index': new_index,
'alias': index_name
}
}]
})
es.indices.update_aliases({
'actions': [
{
'remove': {
'index': new_index,
'alias': settings.ES_INDEXES['new_index']
}
},
]
})
# Compensate for the small race condition in the signal handlers:
# They check if the index exists then updates documents, however when we
# delete the alias in between these two commands, we can end up with
# an auto created index named new_index, so we simply delete it again.
time.sleep(5)
try:
es.indices.delete(settings.ES_INDEXES['new_index'])
except NotFoundError:
pass
if options['queries']:
from django.db import connection
for query in connection.queries:
print query
