def reset_doc_count(self, doc_type, count_key):
count = self.counter.pop(count_key)
if not self.stopper.live_migrate:
couchdb = XFormInstance.get_db()
count = get_doc_count_in_domain_by_type(self.domain, doc_type,
couchdb)
self.statedb.set_counter(doc_type, count)
def _with_progress(self,
doc_types,
iterable,
progress_name='Migrating',
offset_key=None):
doc_count = sum([
get_doc_count_in_domain_by_type(self.domain, doc_type,
XFormInstance.get_db())
for doc_type in doc_types
])
if offset_key is None:
offset = sum(self.counter.get(doc_type) for doc_type in doc_types)
else:
offset = self.counter.get(offset_key)
if self.timing_context:
current_timer = self.timing_context.peek()
current_timer.normalize_denominator = doc_count
if self.with_progress:
prefix = "{} ({})".format(progress_name, ', '.join(doc_types))
return with_progress_bar(iterable,
doc_count,
prefix=prefix,
oneline=False,
offset=offset)
else:
log.info("{} {} ({})".format(progress_name, doc_count,
', '.join(doc_types)))
return iterable
def _with_progress(self,
doc_types,
iterable,
progress_name='Migrating',
offset_key=None):
doc_count = sum([
get_doc_count_in_domain_by_type(self.domain, doc_type,
XFormInstance.get_db())
for doc_type in (d.split(".", 1)[0] for d in doc_types)
])
if offset_key is None:
offset = sum(self.counter.get(doc_type) for doc_type in doc_types)
else:
offset = self.counter.get(offset_key)
self.counter.normalize_timing(doc_count)
if self.with_progress:
prefix = "{} ({})".format(progress_name, ', '.join(doc_types))
return with_progress_bar(iterable,
doc_count,
prefix=prefix,
oneline=False,
offset=offset)
else:
log.info("{} {} ({})".format(progress_name, doc_count,
', '.join(doc_types)))
return iterable
def with_progress(self, doc_type, iterable, count_key):
couchdb = XFormInstance.get_db()
return with_progress_bar(
iterable,
get_doc_count_in_domain_by_type(self.domain, doc_type, couchdb),
prefix=f"Scanning {doc_type}",
offset=self.counter.get(count_key),
oneline="concise",
)
def get_doc_count(model_class, where, entity, domain):
from corehq.apps.domain.dbaccessors import get_doc_count_in_domain_by_type
from ...missingdocs import MissingIds
sql_estimate = estimate_partitioned_row_count(model_class, where)
couchdb = XFormInstance.get_db()
couch_count = sum(
get_doc_count_in_domain_by_type(domain, doc_type, couchdb)
for doc_type in MissingIds.DOC_TYPES[entity])
return min(sql_estimate, couch_count)
def _with_progress(self, doc_types, iterable, progress_name='Migrating'):
if self.with_progress:
doc_count = sum([
get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db())
for doc_type in doc_types
])
prefix = "{} ({})".format(progress_name, ', '.join(doc_types))
return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False)
else:
return iterable
def _with_progress(self, doc_types, iterable, progress_name='Migrating'):
if self.with_progress:
doc_count = sum([
get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db())
for doc_type in doc_types
])
prefix = "{} ({})".format(progress_name, ', '.join(doc_types))
return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False)
else:
return iterable
def _with_progress(self, doc_types, iterable, progress_name='Migrating'):
doc_count = sum([
get_doc_count_in_domain_by_type(self.domain, doc_type, XFormInstance.get_db())
for doc_type in doc_types
])
if self.timing_context:
current_timer = self.timing_context.peek()
current_timer.normalize_denominator = doc_count
if self.with_progress:
prefix = "{} ({})".format(progress_name, ', '.join(doc_types))
return with_progress_bar(iterable, doc_count, prefix=prefix, oneline=False)
else:
self.log_info("{} ({})".format(doc_count, ', '.join(doc_types)))
return iterable
def iter_unmigrated_docs(domain, doc_types, migration_id, counter):
if doc_types != ["XFormInstance"]:
raise NotImplementedError(doc_types)
[doc_type] = doc_types
couch_db = XFormInstance.get_db()
doc_count = counter.pop(doc_type)
if doc_count:
log.info("saved count of %s was %s", doc_type, doc_count)
doc_count = get_doc_count_in_domain_by_type(domain, doc_type, couch_db)
add_docs = partial(counter.add, None, doc_type)
batches = doc_count // iter_id_chunks.chunk_size
iterable = iter_id_chunks(domain, doc_type, migration_id, couch_db)
doc_ids = []
for doc_ids in with_progress_bar(iterable,
batches,
prefix=doc_type,
oneline=False):
yield from iter_docs_not_in_sql(doc_ids, couch_db)
add_docs(len(doc_ids))
def get_couch_doc_count(domain, entity, couchdb):
from corehq.apps.domain.dbaccessors import get_doc_count_in_domain_by_type
return sum(
get_doc_count_in_domain_by_type(domain, doc_type, couchdb)
for doc_type in MissingIds.DOC_TYPES[entity])
def _get_couch_doc_counts(couch_db, domain, doc_types):
counter = Counter()
for doc_type in doc_types:
count = get_doc_count_in_domain_by_type(domain, doc_type, couch_db)
counter.update({doc_type: count})
return counter
def _get_couch_doc_counts(couch_db, domain, doc_types):
counter = Counter()
for doc_type in doc_types:
count = get_doc_count_in_domain_by_type(domain, doc_type, couch_db)
counter.update({doc_type: count})
return counter
