def handle(self, index_name, es2_remote_host, **options): assert index_name in ES_META, "Index name should be one of " + str(ES_META.keys()) self.options = options self.index_info = ES_META[index_name] self.index = self.index_info.index self.es2_remote_host = es2_remote_host # Cancelled/failed reindex queries are tracked to be retried self.cancelled_queries = [] self.es = get_es_export() if index_name not in ["forms", "cases"] and (options.get('start_date') or options.get('end_date')): raise CommandError("start and end dates are supported only for form/case indices") elif index_name in ["forms", "cases"]: start_date = options.get('start_date') end_date = options.get('end_date') else: start_date, end_date = None, None local_count = self.get_es7_count(start_date, end_date) remote_count = self.get_es2_count(start_date, end_date) print("Number of docs in remote ES2 index", remote_count) print("Number of docs in local ES7 index", local_count, "\n") if options.get("print_index_size"): return elif local_count == remote_count: print("Doc counts are same, nothing left to reindex. Exiting!") return else: self.reindex(start_date, end_date)
def add_arguments(self, parser): parser.add_argument( "index_name", help="Index to be reindexed. Should be one of " + ", ".join(ES_META.keys()), ) parser.add_argument( "es2_remote_host", help="Remote ES2 host in http://otherhost:port format" ) parser.add_argument( "--start_date", default=None, help="start date (inclusive) to reindex docs based on inserted_at field." "This is supported only for Form and Case indices. " "In yyyy-MM-dd format. Providing the date range makes reindex run day by day" ) parser.add_argument( "--end_date", default=None, help="end date (inclusive) to reindex docs based on inserted_at field." "This is supported only for Form and Case indices. " "In yyyy-MM-dd format. Providing the date range makes reindex run day by day" ) parser.add_argument( "--batch_size", default=1000, type=int, help="Batch size used by Elasticsearch reindex (default 1000)" ) parser.add_argument( "--print_index_size", action="store_true", default=False, help="Print number of docs in remote and new cluster" )
def __init__(self, index=None): self.index = index if index is not None else self.index if self.index not in ES_META: msg = "%s is not a valid ES index. Available options are: %s" % ( index, ', '.join(ES_META.keys())) raise IndexError(msg) self._default_filters = deepcopy(self.default_filters) self._facets = [] self.es_query = {"query": { "filtered": { "filter": {"and": []}, "query": queries.match_all() } }}
def __init__(self, index=None): self.index = index if index is not None else self.index if self.index not in ES_META: msg = "%s is not a valid ES index. Available options are: %s" % ( index, ', '.join(ES_META.keys())) raise IndexError(msg) self._default_filters = deepcopy(self.default_filters) self._facets = [] self._aggregations = [] self._source = None self.es_query = {"query": { "filtered": { "filter": {"and": []}, "query": queries.match_all() } }}
def __init__(self, index=None, debug_host=None, es_instance_alias=ES_DEFAULT_INSTANCE): from corehq.apps.userreports.util import is_ucr_table self.index = index if index is not None else self.index if self.index not in ES_META and not is_ucr_table(self.index): msg = "%s is not a valid ES index. Available options are: %s" % ( index, ', '.join(ES_META.keys())) raise IndexError(msg) self.debug_host = debug_host self._default_filters = deepcopy(self.default_filters) self._aggregations = [] self._source = None self.es_instance_alias = es_instance_alias self.es_query = {"query": { "filtered": { "filter": {"and": []}, "query": queries.match_all() } }}
def __init__(self, index=None, debug_host=None): from corehq.apps.userreports.util import is_ucr_table self.index = index if index is not None else self.index if self.index not in ES_META and not is_ucr_table(self.index): msg = "%s is not a valid ES index. Available options are: %s" % ( index, ', '.join(ES_META.keys())) raise IndexError(msg) self.debug_host = debug_host self._default_filters = deepcopy(self.default_filters) self._facets = [] self._aggregations = [] self._source = None self.es_query = {"query": { "filtered": { "filter": {"and": []}, "query": queries.match_all() } }}