def reindex(doc_type=None):
'''Reindex models'''
doc_types_names = [m.__name__.lower() for m in adapter_catalog.keys()]
if doc_type and not doc_type.lower() in doc_types_names:
log.error('Unknown type %s', doc_type)
index_name = default_index_name()
log.info('Initiliazing index "{0}"'.format(index_name))
es.initialize(index_name)
for adapter in iter_adapters():
if adapter.doc_type().lower() == doc_type.lower():
index_model(index_name, adapter)
else:
log.info('Copying {0} objects to the new index'.format(
adapter.model.__name__))
# Need upgrade to Elasticsearch-py 5.0.0
# es.reindex({
# 'source': {'index': es.index_name, 'type': adapter.doc_type()},
# 'dest': {'index': index_name}
# })
# http://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.reindex
# This method (introduced in Elasticsearch 2.3 but only in Elasticsearch-py 5.0.0)
# triggers a server-side documents copy.
# Instead we use this helper for meant for backward compatibility
# but with poor performance as copy is client-side (scan+bulk)
es_reindex(es.client,
es.index_name,
index_name,
scan_kwargs={'doc_type': adapter.doc_type()})
set_alias(index_name)
def index(models=None, name=None, force=False, keep=False, timeout=None):
'''
Initialize or rebuild the search index
Models to reindex can optionally be specified as arguments.
If not, all models are reindexed.
'''
index_name = name or default_index_name()
timeout = timeout or current_app.config['ELASTICSEARCH_INDEX_TIMEOUT']
doc_types_names = [m.__name__.lower() for m in adapter_catalog.keys()]
models = [model.lower().rstrip('s') for model in (models or [])]
for model in models:
if model not in doc_types_names:
log.error('Unknown model %s', model)
sys.exit(-1)
log.info('Initiliazing index "%s"', index_name)
if es.indices.exists(index_name):
if IS_TTY and not force:
msg = 'Index {0} will be deleted, are you sure?'
click.confirm(msg.format(index_name), abort=True)
es.indices.delete(index_name, request_timeout=timeout)
es.initialize(index_name)
with handle_error(index_name, keep, timeout):
disable_refresh(index_name, timeout)
for adapter in iter_adapters():
if not models or adapter.doc_type().lower() in models:
index_model(index_name, adapter, timeout)
else:
log.info('Copying %s objects to the new index',
adapter.model.__name__)
# Need upgrade to Elasticsearch-py 5.0.0 to write:
# es.reindex({
# 'source': {'index': es.index_name, 'type': adapter.doc_type()},
# 'dest': {'index': index_name}
# })
#
# http://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.reindex
# This method (introduced in Elasticsearch 2.3 but only in Elasticsearch-py 5.0.0)
# triggers a server-side documents copy.
# Instead we use this helper for meant for backward compatibility
# but with poor performance as copy is client-side (scan+bulk)
es_reindex(es.client,
es.index_name,
index_name,
scan_kwargs={'doc_type': adapter.doc_type()},
bulk_kwargs={'request_timeout': timeout})
enable_refresh(index_name, timeout)
# At this step, we don't want error handler to delete the index
# in case of error
set_alias(index_name, delete=not keep, timeout=timeout)
def index(models=None, name=None, force=False, keep=False):
'''Initialize or rebuild the search index'''
index_name = name or default_index_name()
doc_types_names = [m.__name__.lower() for m in adapter_catalog.keys()]
models = [model.lower().rstrip('s') for model in (models or [])]
for model in models:
if model not in doc_types_names:
log.error('Unknown model %s', model)
sys.exit(-1)
log.info('Initiliazing index "{0}"'.format(index_name))
if es.indices.exists(index_name):
if IS_INTERACTIVE and not force:
msg = 'Index {0} will be deleted, are you sure?'
delete = prompt_bool(msg.format(index_name))
else:
delete = True
if delete:
es.indices.delete(index_name)
else:
sys.exit(-1)
es.initialize(index_name)
with handle_error(index_name, keep):
disable_refresh(index_name)
for adapter in iter_adapters():
if not models or adapter.doc_type().lower() in models:
index_model(index_name, adapter)
else:
log.info('Copying {0} objects to the new index'.format(
adapter.model.__name__))
# Need upgrade to Elasticsearch-py 5.0.0 to write:
# es.reindex({
# 'source': {'index': es.index_name, 'type': adapter.doc_type()},
# 'dest': {'index': index_name}
# })
#
# http://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.reindex
# This method (introduced in Elasticsearch 2.3 but only in Elasticsearch-py 5.0.0)
# triggers a server-side documents copy.
# Instead we use this helper for meant for backward compatibility
# but with poor performance as copy is client-side (scan+bulk)
es_reindex(es.client,
es.index_name,
index_name,
scan_kwargs={'doc_type': adapter.doc_type()})
enable_refresh(index_name)
# At this step, we don't want error handler to delete the index
# in case of error
set_alias(index_name, delete=not keep)
def index(models=None, name=None, force=False, keep=False):
'''
Initialize or rebuild the search index
Models to reindex can optionally be specified as arguments.
If not, all models are reindexed.
'''
index_name = name or default_index_name()
doc_types_names = [m.__name__.lower() for m in adapter_catalog.keys()]
models = [model.lower().rstrip('s') for model in (models or [])]
for model in models:
if model not in doc_types_names:
log.error('Unknown model %s', model)
sys.exit(-1)
log.info('Initiliazing index "{0}"'.format(index_name))
if es.indices.exists(index_name):
if IS_TTY and not force:
msg = 'Index {0} will be deleted, are you sure?'
click.confirm(msg.format(index_name), abort=True)
es.indices.delete(index_name)
es.initialize(index_name)
with handle_error(index_name, keep):
disable_refresh(index_name)
for adapter in iter_adapters():
if not models or adapter.doc_type().lower() in models:
index_model(index_name, adapter)
else:
log.info('Copying {0} objects to the new index'.format(
adapter.model.__name__))
# Need upgrade to Elasticsearch-py 5.0.0 to write:
# es.reindex({
# 'source': {'index': es.index_name, 'type': adapter.doc_type()},
# 'dest': {'index': index_name}
# })
#
# http://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.reindex
# This method (introduced in Elasticsearch 2.3 but only in Elasticsearch-py 5.0.0)
# triggers a server-side documents copy.
# Instead we use this helper for meant for backward compatibility
# but with poor performance as copy is client-side (scan+bulk)
es_reindex(es.client, es.index_name, index_name, scan_kwargs={
'doc_type': adapter.doc_type()
})
enable_refresh(index_name)
# At this step, we don't want error handler to delete the index
# in case of error
set_alias(index_name, delete=not keep)
def doReindex(es, index, settings):
"""
will create a new index, copy the mapping from the old index, and then copy the data from old index to the new index.
This is useful for changing the default template, etc.
:param es:
:param index: the name of the old index
:param settings: settings object returned from a GET index request
:return:
"""
today = date.today().strftime("%Y%m%d")
projectId = index.split("_")[0]
data = {}
data['mappings'] = settings['mappings']
es.indices.create(projectId + "_" + today, body=json.dumps(data))
es_reindex(es, index, projectId + "_" + today, chunk_size=500)