def reset_doc_count(self, doc_type, count_key): count = self.counter.pop(count_key) if not self.stopper.live_migrate: couchdb = XFormInstance.get_db() count = get_doc_count_in_domain_by_type(self.domain, doc_type, couchdb) self.statedb.set_counter(doc_type, count)
def _with_progress(self, doc_types, iterable, progress_name='Migrating', offset_key=None): doc_count = sum([ get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db()) for doc_type in doc_types ]) if offset_key is None: offset = sum(self.counter.get(doc_type) for doc_type in doc_types) else: offset = self.counter.get(offset_key) if self.timing_context: current_timer = self.timing_context.peek() current_timer.normalize_denominator = doc_count if self.with_progress: prefix = "{} ({})".format(progress_name, ', '.join(doc_types)) return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False, offset=offset) else: log.info("{} {} ({})".format(progress_name, doc_count, ', '.join(doc_types))) return iterable
def _with_progress(self, doc_types, iterable, progress_name='Migrating', offset_key=None): doc_count = sum([ get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db()) for doc_type in (d.split(".", 1)[0] for d in doc_types) ]) if offset_key is None: offset = sum(self.counter.get(doc_type) for doc_type in doc_types) else: offset = self.counter.get(offset_key) self.counter.normalize_timing(doc_count) if self.with_progress: prefix = "{} ({})".format(progress_name, ', '.join(doc_types)) return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False, offset=offset) else: log.info("{} {} ({})".format(progress_name, doc_count, ', '.join(doc_types))) return iterable
def with_progress(self, doc_type, iterable, count_key): couchdb = XFormInstance.get_db() return with_progress_bar( iterable, get_doc_count_in_domain_by_type(self.domain, doc_type, couchdb), prefix=f"Scanning {doc_type}", offset=self.counter.get(count_key), oneline="concise", )
def get_doc_count(model_class, where, entity, domain): from corehq.apps.domain.dbaccessors import get_doc_count_in_domain_by_type from ...missingdocs import MissingIds sql_estimate = estimate_partitioned_row_count(model_class, where) couchdb = XFormInstance.get_db() couch_count = sum( get_doc_count_in_domain_by_type(domain, doc_type, couchdb) for doc_type in MissingIds.DOC_TYPES[entity]) return min(sql_estimate, couch_count)
def _with_progress(self, doc_types, iterable, progress_name='Migrating'): if self.with_progress: doc_count = sum([ get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db()) for doc_type in doc_types ]) prefix = "{} ({})".format(progress_name, ', '.join(doc_types)) return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False) else: return iterable
def _with_progress(self, doc_types, iterable, progress_name='Migrating'): if self.with_progress: doc_count = sum([ get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db()) for doc_type in doc_types ]) prefix = "{} ({})".format(progress_name, ', '.join(doc_types)) return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False) else: return iterable
def _with_progress(self, doc_types, iterable, progress_name='Migrating'): doc_count = sum([ get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db()) for doc_type in doc_types ]) if self.timing_context: current_timer = self.timing_context.peek() current_timer.normalize_denominator = doc_count if self.with_progress: prefix = "{} ({})".format(progress_name, ', '.join(doc_types)) return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False) else: self.log_info("{} ({})".format(doc_count, ', '.join(doc_types))) return iterable
def iter_unmigrated_docs(domain, doc_types, migration_id, counter): if doc_types != ["XFormInstance"]: raise NotImplementedError(doc_types) [doc_type] = doc_types couch_db = XFormInstance.get_db() doc_count = counter.pop(doc_type) if doc_count: log.info("saved count of %s was %s", doc_type, doc_count) doc_count = get_doc_count_in_domain_by_type(domain, doc_type, couch_db) add_docs = partial(counter.add, None, doc_type) batches = doc_count // iter_id_chunks.chunk_size iterable = iter_id_chunks(domain, doc_type, migration_id, couch_db) doc_ids = [] for doc_ids in with_progress_bar(iterable, batches, prefix=doc_type, oneline=False): yield from iter_docs_not_in_sql(doc_ids, couch_db) add_docs(len(doc_ids))
def get_couch_doc_count(domain, entity, couchdb): from corehq.apps.domain.dbaccessors import get_doc_count_in_domain_by_type return sum( get_doc_count_in_domain_by_type(domain, doc_type, couchdb) for doc_type in MissingIds.DOC_TYPES[entity])
def _get_couch_doc_counts(couch_db, domain, doc_types): counter = Counter() for doc_type in doc_types: count = get_doc_count_in_domain_by_type(domain, doc_type, couch_db) counter.update({doc_type: count}) return counter
def _get_couch_doc_counts(couch_db, domain, doc_types): counter = Counter() for doc_type in doc_types: count = get_doc_count_in_domain_by_type(domain, doc_type, couch_db) counter.update({doc_type: count}) return counter