def handle(self, index=None, reset=False, set_for_usage=False, **kwargs):
es = get_es_new()
if reset and not set_for_usage:
confirm = input(
"""
Are you sure you want to want to delete existing indices
Note that this will delete any data in them. y/N?
"""
)
if confirm != "y":
print("Cancelled by user.")
return
if index:
indices = [registry_entry(index)]
else:
indices = get_registry().values()
for index in indices:
if set_for_usage:
prepare_index_for_usage(es, index)
else:
if reset:
clean_index(es, index)
prepare_index_for_reindex(es, index)
if set_for_usage:
print("index ready for usage")
else:
print(
"""Initialized all indices and applied reindex settings
After reindex is finished, you can run this command again
with --set-for-usage to remove reindex settings and make it
ready for usage.
"""
)
def run_query(index_cname, q, for_export=False):
index_info = registry_entry(index_cname)
adapter = doc_adapter_from_info(index_info, for_export)
try:
results = adapter.search(q)
except ElasticsearchException as e:
raise ESError(e)
return results
def handle(self, source_index, target_index_name, **options):
es = get_es_export()
if not es.indices.exists(source_index):
raise CommandError(
f"Source index does not exist: '{source_index}'")
target_index_info = registry_entry(target_index_name)
target_index = target_index_info.index
_initialize_target(es, target_index_info)
source_count = _get_doc_count(es, source_index)
target_count = _get_doc_count(es, target_index)
print("Number of docs in source index", source_count)
print("Number of docs in target index", target_count)
if options.get("print_index_size"):
return
elif source_count == target_count:
print("Doc counts are same, nothing left to reindex. Exiting!")
return
if not options["monitor_task_id"]:
print(
f"Starting reindex for index from '{source_index}' to '{target_index}'"
)
task_id = start_reindex(es, source_index, target_index)
else:
task_id = options["monitor_task_id"]
check_task_progress(es, task_id)
print("\nReindex task complete.")
source_count = _get_doc_count(es, source_index)
target_count = _get_doc_count(es, target_index)
print("Number of docs in source index", source_count)
print("Number of docs in target index", target_count)
print("")
print(
inspect.cleandoc(f"""
Perform manual checks to verify the reindex is complete.
Once you are satisfied that the reindex is complete you should run the following:
./manage.py initialize_es_indices --index {target_index_name} --set-for-usage
"""))
def scroll_query(index_cname, query, for_export=False, **kw):
"""Perfrom a scrolling search, yielding each doc until the entire context
is exhausted.
:param index_cname: Canonical (registered) name of index to search.
:param query: Dict, raw search query.
:param for_export: See `corehq.apps.es.client.get_client()`
:param **kw: Additional scroll keyword arguments. Valid options:
`size`: Integer, scroll size (number of documents per "scroll")
`scroll`: String, time value specifying how long the Elastic
cluster should keep the search context alive.
"""
valid_kw = {"size", "scroll"}
if not set(kw).issubset(valid_kw):
raise ValueError(f"invalid keyword args: {set(kw) - valid_kw}")
index_info = registry_entry(index_cname)
adapter = doc_adapter_from_info(index_info, for_export=for_export)
try:
yield from adapter.scroll(query, **kw)
except ElasticsearchException as e:
raise ESError(e)
def send_to_elasticsearch(index_cname,
doc,
delete=False,
es_merge_update=False):
"""
Utility method to update the doc in elasticsearch.
Duplicates the functionality of pillowtop but can be called directly.
"""
doc_id = doc['_id']
if isinstance(doc_id, bytes):
doc_id = doc_id.decode('utf-8')
index_info = registry_entry(index_cname)
return send_to_es(
index_info=index_info,
doc_type=index_info.type,
doc_id=doc_id,
es_getter=get_client,
name="{}.{} <{}>:".format(send_to_elasticsearch.__module__,
send_to_elasticsearch.__name__, index_cname),
data=doc,
delete=delete,
es_merge_update=es_merge_update,
)
def _iter_es_docs(index_cname, ids):
"""Returns a generator which pulls documents from elasticsearch in chunks"""
index_info = registry_entry(index_cname)
adapter = doc_adapter_from_info(index_info)
yield from adapter.iter_docs(ids)
def refresh_elasticsearch_index(index_cname):
index_info = registry_entry(index_cname)
ElasticManageAdapter().index_refresh(index_info.alias)
def count_query(index_cname, q):
index_info = registry_entry(index_cname)
adapter = doc_adapter_from_info(index_info)
return adapter.count(q)