def _print_status(self, doc_type, counts, statedb, short, diffs_only):
has_diffs = counts.missing or counts.diffs
if diffs_only and not has_diffs:
return False
if not short:
print("_" * 40)
ndiff = shell_red(f"{counts.diffs if counts.diffs else '':>8}")
miss = shell_red(f"{counts.missing if counts.missing else '':>8}")
chg = counts.changes if counts.changes else ""
print(f"{doc_type:<22} {counts.total:>9} {ndiff} {miss} {chg:>8}")
if not short:
# print ids found in Couch but not in SQL
i = 0
missing_ids = statedb.iter_missing_doc_ids(doc_type)
for i, missing_id in enumerate(missing_ids, start=1):
print(missing_id)
assert i == counts.missing, (i, counts.missing)
return has_diffs
def _write_output(self, template, output_rows, with_header_divider=True, with_color=True):
self.stdout.write(template.format('Doc Type', 'Couch', 'SQL', 'ES'))
if with_header_divider:
self.stdout.write(template.format('-' * 50, *['-' * 20] * 3))
for doc_type, couch_count, sql_count, es_count in output_rows:
row_template = template
couch_dff = couch_count and couch_count != es_count
sql_diff = sql_count and sql_count != es_count
if with_color and es_count and (couch_dff or sql_diff):
row_template = shell_red(template)
self.stdout.write(row_template.format(doc_type, couch_count, sql_count, es_count))
def _print_status(self, doc_type, counts, statedb, short, diffs_only):
has_diffs = counts.missing or counts.diffs
if diffs_only and not has_diffs:
return False
if not short:
print("_" * 40)
stats = [
f"{doc_type:<22} {counts.total:>9} docs",
shell_red(f"{counts.missing} missing from SQL") if counts.missing else "",
shell_red(f"{counts.diffs} have diffs") if counts.diffs else "",
]
print(", ".join(x for x in stats if x))
if not short:
# print ids found in Couch but not in SQL
i = 0
missing_ids = statedb.iter_missing_doc_ids(doc_type)
for i, missing_id in enumerate(missing_ids, start=1):
print(missing_id)
assert i == counts.missing, (i, counts.missing)
return has_diffs
def _write_output(self,
template,
output_rows,
with_header_divider=True,
with_color=True):
self.stdout.write(template.format('Doc Type', 'Couch', 'SQL', 'ES'))
if with_header_divider:
self.stdout.write(template.format('-' * 50, *['-' * 20] * 3))
for doc_type, couch_count, sql_count, es_count in output_rows:
row_template = template
couch_dff = couch_count and couch_count != es_count
sql_diff = sql_count and sql_count != es_count
if with_color and es_count and (couch_dff or sql_diff):
row_template = shell_red(template)
self.stdout.write(
row_template.format(doc_type, couch_count, sql_count,
es_count))
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
if not self.live_migrate:
self.live_migrate = status == MigrationStatus.DRY_RUN
if self.missing_docs == RECHECK:
recheck_missing_docs(domain, self.state_dir)
elif self.missing_docs != CACHED:
resume = self.missing_docs == RESUME
find_missing_docs(domain, self.state_dir, self.live_migrate,
resume)
print(f"Couch to SQL migration status for {domain}: {status}")
try:
statedb = open_state_db(domain, self.state_dir)
except NotFoundError:
self.print_couch_stats(domain)
return
doc_counts = statedb.get_doc_counts()
has_diffs = False
ZERO = Counts()
print(f"{'':<22} Docs Diffs Missing Changes")
for doc_type in (list(doc_types()) +
["HQSubmission", "XFormInstance-Deleted"] +
CASE_DOC_TYPES):
has_diffs |= self._print_status(
doc_type,
doc_counts.get(doc_type, ZERO),
statedb,
short,
diffs_only,
)
if any(x.missing for x in doc_counts.values()):
print("\nRun again with --forms=missing to migrate missing docs")
pending = statedb.count_undiffed_cases()
if pending:
print(shell_red(f"\nThere are {pending} case diffs pending."))
print(
f"Resolution: couch_sql_diff {domain} cases --select=pending")
return True
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def _highlight(text):
return shell_red(text) if has_diff else text
def _highlight(text):
return shell_red(text) if has_diff else text
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = open_state_db(domain, self.state_dir)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
if doc_type == "XFormInstance":
form_ids_in_couch.update(
get_doc_ids_in_domain_by_type(domain, "HQSubmission",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
ZERO = Counts(0, 0)
if db.has_doc_counts():
doc_counts = db.get_doc_counts()
couch_missing_cases = doc_counts.get("CommCareCase-couch",
ZERO).missing
else:
doc_counts = None
couch_missing_cases = 0
for doc_type in CASE_DOC_TYPES:
if doc_counts is not None:
counts = doc_counts.get(doc_type, ZERO)
case_ids_in_couch = db.get_missing_doc_ids(
doc_type) if counts.missing else set()
case_ids_in_sql = counts
elif doc_type == "CommCareCase":
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(
CaseAccessorSQL.get_case_ids_in_domain(domain))
elif doc_type == "CommCareCase-Deleted":
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
else:
raise NotImplementedError(doc_type)
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type,
case_ids_in_couch,
case_ids_in_sql,
diff_count,
num_docs_with_diffs,
short,
diffs_only,
)
if doc_type == "CommCareCase" and couch_missing_cases:
has_diffs = True
print(
shell_red("%s cases could not be loaded from Couch" %
couch_missing_cases))
if not short:
for case_id in db.get_missing_doc_ids(
"CommCareCase-couch"):
print(case_id)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
