def handle(self, index=None, reset=False, set_for_usage=False, **kwargs):
es = get_es_new()
if reset and not set_for_usage:
confirm = input(
"""
Are you sure you want to want to delete existing indices
Note that this will delete any data in them. y/N?
"""
)
if confirm != "y":
print("Cancelled by user.")
return
if index:
indices = [ES_META[index]]
else:
indices = ES_META.values()
for index in indices:
if set_for_usage:
prepare_index_for_usage(es, index)
else:
if reset:
clean_index(es, index)
prepare_index_for_reindex(es, index)
if set_for_usage:
print("index ready for usage")
else:
print(
"""Initialized all indices and applied reindex settings
After reindex is finished, you can run this command again
with --set-for-usage to remove reindex settings and make it
ready for usage.
"""
)
def handle(self, index_name, es2_remote_host, **options):
assert index_name in ES_META, "Index name should be one of " + str(ES_META.keys())
self.options = options
self.index_info = ES_META[index_name]
self.index = self.index_info.index
self.es2_remote_host = es2_remote_host
# Cancelled/failed reindex queries are tracked to be retried
self.cancelled_queries = []
self.es = get_es_export()
if index_name not in ["forms", "cases"] and (options.get('start_date') or options.get('end_date')):
raise CommandError("start and end dates are supported only for form/case indices")
elif index_name in ["forms", "cases"]:
start_date = options.get('start_date')
end_date = options.get('end_date')
else:
start_date, end_date = None, None
local_count = self.get_es7_count(start_date, end_date)
remote_count = self.get_es2_count(start_date, end_date)
print("Number of docs in remote ES2 index", remote_count)
print("Number of docs in local ES7 index", local_count, "\n")
if options.get("print_index_size"):
return
elif local_count == remote_count:
print("Doc counts are same, nothing left to reindex. Exiting!")
return
else:
self.reindex(start_date, end_date)
def add_arguments(self, parser):
parser.add_argument(
"index_name",
help="Index to be reindexed. Should be one of " + ", ".join(ES_META.keys()),
)
parser.add_argument(
"es2_remote_host",
help="Remote ES2 host in http://otherhost:port format"
)
parser.add_argument(
"--start_date",
default=None,
help="start date (inclusive) to reindex docs based on inserted_at field."
"This is supported only for Form and Case indices. "
"In yyyy-MM-dd format. Providing the date range makes reindex run day by day"
)
parser.add_argument(
"--end_date",
default=None,
help="end date (inclusive) to reindex docs based on inserted_at field."
"This is supported only for Form and Case indices. "
"In yyyy-MM-dd format. Providing the date range makes reindex run day by day"
)
parser.add_argument(
"--batch_size",
default=1000,
type=int,
help="Batch size used by Elasticsearch reindex (default 1000)"
)
parser.add_argument(
"--print_index_size",
action="store_true",
default=False,
help="Print number of docs in remote and new cluster"
)
def _verify_is_alias(self, index_or_alias):
from corehq.elastic import ES_META, ESError
from pillowtop.tests.utils import TEST_ES_ALIAS
all_es_aliases = [index_info.alias for index_info in ES_META.values()] + [TEST_ES_ALIAS]
if index_or_alias not in all_es_aliases:
raise ESError(
f"{index_or_alias} is an unknown alias, query target must be one of {all_es_aliases}")
def _verify_is_alias(self, index_or_alias):
from corehq.elastic import ES_META
if settings.ENABLE_ES_INTERFACE_LOGGING:
logger = logging.getLogger('es_interface')
all_es_aliases = [index_info.alias for index_info in ES_META.values()]
if index_or_alias not in all_es_aliases:
logger.info("Found a use case where an index is queried instead of alias")
logger.info(traceback.format_stack())
def __init__(self, index=None):
self.index = index if index is not None else self.index
if self.index not in ES_META:
msg = "%s is not a valid ES index. Available options are: %s" % (
index, ', '.join(ES_META.keys()))
raise IndexError(msg)
self._default_filters = deepcopy(self.default_filters)
self._facets = []
self.es_query = {"query": {
"filtered": {
"filter": {"and": []},
"query": queries.match_all()
}
}}
def __init__(self, index=None):
self.index = index if index is not None else self.index
if self.index not in ES_META:
msg = "%s is not a valid ES index. Available options are: %s" % (
index, ', '.join(ES_META.keys()))
raise IndexError(msg)
self._default_filters = deepcopy(self.default_filters)
self._facets = []
self._aggregations = []
self._source = None
self.es_query = {"query": {
"filtered": {
"filter": {"and": []},
"query": queries.match_all()
}
}}
def __init__(self, index=None, debug_host=None, es_instance_alias=ES_DEFAULT_INSTANCE):
from corehq.apps.userreports.util import is_ucr_table
self.index = index if index is not None else self.index
if self.index not in ES_META and not is_ucr_table(self.index):
msg = "%s is not a valid ES index. Available options are: %s" % (
index, ', '.join(ES_META.keys()))
raise IndexError(msg)
self.debug_host = debug_host
self._default_filters = deepcopy(self.default_filters)
self._aggregations = []
self._source = None
self.es_instance_alias = es_instance_alias
self.es_query = {"query": {
"filtered": {
"filter": {"and": []},
"query": queries.match_all()
}
}}
def __init__(self, index=None, debug_host=None):
from corehq.apps.userreports.util import is_ucr_table
self.index = index if index is not None else self.index
if self.index not in ES_META and not is_ucr_table(self.index):
msg = "%s is not a valid ES index. Available options are: %s" % (
index, ', '.join(ES_META.keys()))
raise IndexError(msg)
self.debug_host = debug_host
self._default_filters = deepcopy(self.default_filters)
self._facets = []
self._aggregations = []
self._source = None
self.es_query = {"query": {
"filtered": {
"filter": {"and": []},
"query": queries.match_all()
}
}}