def do_MIGRATE(self, domain):
if self.finish:
assert not self.live_migrate, "--live and --finish are mutually exclusive"
elif not self.live_migrate:
status = get_couch_sql_migration_status(domain)
if status == MigrationStatus.DRY_RUN:
log.info(
"Continuing live migration. Use --finish to complete.")
self.live_migrate = True
if self.missing_docs == CACHED:
self.missing_docs = RESUME
if self.forms:
if not couch_sql_migration_in_progress(domain):
log.error("cannot migrate specific forms: migration is %s",
get_couch_sql_migration_status(domain))
sys.exit(1)
else:
set_couch_sql_migration_started(domain, self.live_migrate)
do_couch_to_sql_migration(
domain,
self.state_dir,
with_progress=not self.no_input,
live_migrate=self.live_migrate,
case_diff=self.case_diff,
rebuild_state=self.rebuild_state,
stop_on_error=self.stop_on_error,
forms=self.forms,
)
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
if self.live_migrate:
print("Live migration completed.")
if has_diffs:
print("\nRun `diff` or `stats [--verbose]` for more details.\n")
sys.exit(1)
def do_MIGRATE(self, domain):
if self.finish:
assert not self.live_migrate, "--live and --finish are mutually exclusive"
elif not self.live_migrate:
status = get_couch_sql_migration_status(domain)
if status == MigrationStatus.DRY_RUN:
log.info(
"Continuing live migration. Use --finish to complete.")
self.live_migrate = True
set_couch_sql_migration_started(domain, self.live_migrate)
do_couch_to_sql_migration(
domain,
self.state_dir,
with_progress=not self.no_input,
live_migrate=self.live_migrate,
diff_process=CASE_DIFF[self.case_diff],
rebuild_state=self.rebuild_state,
stop_on_error=self.stop_on_error,
forms=self.forms,
)
return_code = 0
if self.live_migrate:
print("Live migration completed.")
has_diffs = True
else:
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
return_code = int(has_diffs)
if has_diffs:
print("\nRun `diff` or `stats [--verbose]` for more details.\n")
if return_code:
sys.exit(return_code)
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
if not self.live_migrate:
self.live_migrate = status == MigrationStatus.DRY_RUN
if self.missing_docs != CACHED:
resume = self.missing_docs == RESUME
find_missing_docs(domain, self.state_dir, self.live_migrate, resume)
print("Couch to SQL migration status for {}: {}".format(domain, status))
statedb = open_state_db(domain, self.state_dir)
doc_counts = statedb.get_doc_counts()
has_diffs = False
ZERO = Counts()
for doc_type in (
list(doc_types())
+ ["HQSubmission", "XFormInstance-Deleted"]
+ CASE_DOC_TYPES
):
has_diffs |= self._print_status(
doc_type,
doc_counts.get(doc_type, ZERO),
statedb,
short,
diffs_only,
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
if not self.live_migrate:
self.live_migrate = status == MigrationStatus.DRY_RUN
if self.missing_docs == RECHECK:
recheck_missing_docs(domain, self.state_dir)
elif self.missing_docs != CACHED:
resume = self.missing_docs == RESUME
find_missing_docs(domain, self.state_dir, self.live_migrate,
resume)
print(f"Couch to SQL migration status for {domain}: {status}")
statedb = open_state_db(domain, self.state_dir)
doc_counts = statedb.get_doc_counts()
has_diffs = False
ZERO = Counts()
print(f"{'':<22} Docs Diffs Missing Changes")
for doc_type in (list(doc_types()) +
["HQSubmission", "XFormInstance-Deleted"] +
CASE_DOC_TYPES):
has_diffs |= self._print_status(
doc_type,
doc_counts.get(doc_type, ZERO),
statedb,
short,
diffs_only,
)
pending = statedb.count_undiffed_cases()
if pending:
print(shell_red(f"\nThere are {pending} case diffs pending."))
print(
f"Resolution: couch_sql_diff cases {domain} --select=pending")
return True
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = open_state_db(domain, self.state_dir)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
if doc_type == "XFormInstance":
form_ids_in_couch.update(
get_doc_ids_in_domain_by_type(domain, "HQSubmission",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
ZERO = Counts(0, 0)
if db.has_doc_counts():
doc_counts = db.get_doc_counts()
couch_missing_cases = doc_counts.get("CommCareCase-couch",
ZERO).missing
else:
doc_counts = None
couch_missing_cases = 0
for doc_type in CASE_DOC_TYPES:
if doc_counts is not None:
counts = doc_counts.get(doc_type, ZERO)
case_ids_in_couch = db.get_missing_doc_ids(
doc_type) if counts.missing else set()
case_ids_in_sql = counts
elif doc_type == "CommCareCase":
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(
CaseAccessorSQL.get_case_ids_in_domain(domain))
elif doc_type == "CommCareCase-Deleted":
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
else:
raise NotImplementedError(doc_type)
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type,
case_ids_in_couch,
case_ids_in_sql,
diff_count,
num_docs_with_diffs,
short,
diffs_only,
)
if doc_type == "CommCareCase" and couch_missing_cases:
has_diffs = True
print(
shell_red("%s cases could not be loaded from Couch" %
couch_missing_cases))
if not short:
for case_id in db.get_missing_doc_ids(
"CommCareCase-couch"):
print(case_id)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs