def print_stats(self, domain, short=True, diffs_only=False):
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count = diff_stats.pop(doc_type, 0)
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count = diff_stats.pop("XFormInstance-Deleted", 0)
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, short, diffs_only)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count = diff_stats.pop("CommCareCase", 0)
has_diffs |= self._print_status('CommCareCase', case_ids_in_couch,
case_ids_in_sql, diff_count, short,
diffs_only)
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count = diff_stats.pop("CommCareCase-Deleted", 0)
has_diffs |= self._print_status('CommCareCase-Deleted',
case_ids_in_couch, case_ids_in_sql,
diff_count, short, diffs_only)
if diff_stats:
for key, count in diff_stats.items():
has_diffs |= self._print_status(key, set(), set(), count,
short, diffs_only)
if diffs_only and not has_diffs:
print shell_green("No differences found between old and new docs!")
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count = diff_stats.pop(doc_type, 0)
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count = diff_stats.pop("XFormInstance-Deleted", 0)
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count = diff_stats.pop("CommCareCase", 0)
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count = diff_stats.pop("CommCareCase-Deleted", 0)
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, short, diffs_only
)
if diff_stats:
for key, count in diff_stats.items():
has_diffs |= self._print_status(
key, set(), set(), count, short, diffs_only
)
if diffs_only and not has_diffs:
print shell_green("No differences found between old and new docs!")
return has_diffs
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write(
"{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
if not self.live_migrate:
self.live_migrate = status == MigrationStatus.DRY_RUN
if self.missing_docs != CACHED:
resume = self.missing_docs == RESUME
find_missing_docs(domain, self.state_dir, self.live_migrate, resume)
print("Couch to SQL migration status for {}: {}".format(domain, status))
statedb = open_state_db(domain, self.state_dir)
doc_counts = statedb.get_doc_counts()
has_diffs = False
ZERO = Counts()
for doc_type in (
list(doc_types())
+ ["HQSubmission", "XFormInstance-Deleted"]
+ CASE_DOC_TYPES
):
has_diffs |= self._print_status(
doc_type,
doc_counts.get(doc_type, ZERO),
statedb,
short,
diffs_only,
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return
set_couch_sql_migration_started(domain)
with SignalHandlerContext([signal.SIGTERM, signal.SIGINT],
_get_sigterm_handler(domain)):
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
else:
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green(
"Domain migrated: {}".format(domain)))
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write("{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write("Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table(['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [
(doc_type,) + stat for doc_type, stat in stats.items()
])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
if not self.live_migrate:
self.live_migrate = status == MigrationStatus.DRY_RUN
if self.missing_docs == RECHECK:
recheck_missing_docs(domain, self.state_dir)
elif self.missing_docs != CACHED:
resume = self.missing_docs == RESUME
find_missing_docs(domain, self.state_dir, self.live_migrate,
resume)
print(f"Couch to SQL migration status for {domain}: {status}")
try:
statedb = open_state_db(domain, self.state_dir)
except NotFoundError:
self.print_couch_stats(domain)
return
doc_counts = statedb.get_doc_counts()
has_diffs = False
ZERO = Counts()
print(f"{'':<22} Docs Diffs Missing Changes")
for doc_type in (list(doc_types()) +
["HQSubmission", "XFormInstance-Deleted"] +
CASE_DOC_TYPES):
has_diffs |= self._print_status(
doc_type,
doc_counts.get(doc_type, ZERO),
statedb,
short,
diffs_only,
)
if any(x.missing for x in doc_counts.values()):
print("\nRun again with --forms=missing to migrate missing docs")
pending = statedb.count_undiffed_cases()
if pending:
print(shell_red(f"\nThere are {pending} case diffs pending."))
print(
f"Resolution: couch_sql_diff {domain} cases --select=pending")
return True
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = open_state_db(domain, self.state_dir)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
if doc_type == "XFormInstance":
form_ids_in_couch.update(
get_doc_ids_in_domain_by_type(domain, "HQSubmission",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
ZERO = Counts(0, 0)
if db.has_doc_counts():
doc_counts = db.get_doc_counts()
couch_missing_cases = doc_counts.get("CommCareCase-couch",
ZERO).missing
else:
doc_counts = None
couch_missing_cases = 0
for doc_type in CASE_DOC_TYPES:
if doc_counts is not None:
counts = doc_counts.get(doc_type, ZERO)
case_ids_in_couch = db.get_missing_doc_ids(
doc_type) if counts.missing else set()
case_ids_in_sql = counts
elif doc_type == "CommCareCase":
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(
CaseAccessorSQL.get_case_ids_in_domain(domain))
elif doc_type == "CommCareCase-Deleted":
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
else:
raise NotImplementedError(doc_type)
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type,
case_ids_in_couch,
case_ids_in_sql,
diff_count,
num_docs_with_diffs,
short,
diffs_only,
)
if doc_type == "CommCareCase" and couch_missing_cases:
has_diffs = True
print(
shell_red("%s cases could not be loaded from Couch" %
couch_missing_cases))
if not short:
for case_id in db.get_missing_doc_ids(
"CommCareCase-couch"):
print(case_id)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
