def print_stats(self, domain, short=True, diffs_only=False): db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type)) diff_count = diff_stats.pop(doc_type, 0) has_diffs |= self._print_status(doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, short, diffs_only) form_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count = diff_stats.pop("XFormInstance-Deleted", 0) has_diffs |= self._print_status("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, short, diffs_only) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count = diff_stats.pop("CommCareCase", 0) has_diffs |= self._print_status('CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, short, diffs_only) case_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db())) case_ids_in_sql = set( CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count = diff_stats.pop("CommCareCase-Deleted", 0) has_diffs |= self._print_status('CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, short, diffs_only) if diff_stats: for key, count in diff_stats.items(): has_diffs |= self._print_status(key, set(), set(), count, short, diffs_only) if diffs_only and not has_diffs: print shell_green("No differences found between old and new docs!") return has_diffs
def print_stats(self, domain, short=True, diffs_only=False): db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)) diff_count = diff_stats.pop(doc_type, 0) has_diffs |= self._print_status( doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, short, diffs_only ) form_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) ) form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count = diff_stats.pop("XFormInstance-Deleted", 0) has_diffs |= self._print_status( "XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, short, diffs_only ) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count = diff_stats.pop("CommCareCase", 0) has_diffs |= self._print_status( 'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, short, diffs_only ) case_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) ) case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count = diff_stats.pop("CommCareCase-Deleted", 0) has_diffs |= self._print_status( 'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, short, diffs_only ) if diff_stats: for key, count in diff_stats.items(): has_diffs |= self._print_status( key, set(), set(), count, short, diffs_only ) if diffs_only and not has_diffs: print shell_green("No differences found between old and new docs!") return has_diffs
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): self.stderr.write( "{} migration is already in progress".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write( "Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table([ 'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs' ], [(doc_type, ) + stat for doc_type, stat in stats.items()]) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green("Domain migrated: {}".format(domain))) return True, None
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) if not self.live_migrate: self.live_migrate = status == MigrationStatus.DRY_RUN if self.missing_docs != CACHED: resume = self.missing_docs == RESUME find_missing_docs(domain, self.state_dir, self.live_migrate, resume) print("Couch to SQL migration status for {}: {}".format(domain, status)) statedb = open_state_db(domain, self.state_dir) doc_counts = statedb.get_doc_counts() has_diffs = False ZERO = Counts() for doc_type in ( list(doc_types()) + ["HQSubmission", "XFormInstance-Deleted"] + CASE_DOC_TYPES ): has_diffs |= self._print_status( doc_type, doc_counts.get(doc_type, ZERO), statedb, short, diffs_only, ) if diffs_only and not has_diffs: print(shell_green("No differences found between old and new docs!")) return has_diffs
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return set_couch_sql_migration_started(domain) with SignalHandlerContext([signal.SIGTERM, signal.SIGINT], _get_sigterm_handler(domain)): do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write( "Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table([ 'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs' ], [(doc_type, ) + stat for doc_type, stat in stats.items()]) else: assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green( "Domain migrated: {}".format(domain)))
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): self.stderr.write("{} migration is already in progress".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write("Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table(['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [ (doc_type,) + stat for doc_type, stat in stats.items() ]) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green("Domain migrated: {}".format(domain))) return True, None
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) if not self.live_migrate: self.live_migrate = status == MigrationStatus.DRY_RUN if self.missing_docs == RECHECK: recheck_missing_docs(domain, self.state_dir) elif self.missing_docs != CACHED: resume = self.missing_docs == RESUME find_missing_docs(domain, self.state_dir, self.live_migrate, resume) print(f"Couch to SQL migration status for {domain}: {status}") try: statedb = open_state_db(domain, self.state_dir) except NotFoundError: self.print_couch_stats(domain) return doc_counts = statedb.get_doc_counts() has_diffs = False ZERO = Counts() print(f"{'':<22} Docs Diffs Missing Changes") for doc_type in (list(doc_types()) + ["HQSubmission", "XFormInstance-Deleted"] + CASE_DOC_TYPES): has_diffs |= self._print_status( doc_type, doc_counts.get(doc_type, ZERO), statedb, short, diffs_only, ) if any(x.missing for x in doc_counts.values()): print("\nRun again with --forms=missing to migrate missing docs") pending = statedb.count_undiffed_cases() if pending: print(shell_red(f"\nThere are {pending} case diffs pending.")) print( f"Resolution: couch_sql_diff {domain} cases --select=pending") return True if diffs_only and not has_diffs: print( shell_green("No differences found between old and new docs!")) return has_diffs
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format( domain, status)) db = open_state_db(domain, self.state_dir) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) if doc_type == "XFormInstance": form_ids_in_couch.update( get_doc_ids_in_domain_by_type(domain, "HQSubmission", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status(doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) form_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) ZERO = Counts(0, 0) if db.has_doc_counts(): doc_counts = db.get_doc_counts() couch_missing_cases = doc_counts.get("CommCareCase-couch", ZERO).missing else: doc_counts = None couch_missing_cases = 0 for doc_type in CASE_DOC_TYPES: if doc_counts is not None: counts = doc_counts.get(doc_type, ZERO) case_ids_in_couch = db.get_missing_doc_ids( doc_type) if counts.missing else set() case_ids_in_sql = counts elif doc_type == "CommCareCase": case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set( CaseAccessorSQL.get_case_ids_in_domain(domain)) elif doc_type == "CommCareCase-Deleted": case_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db())) case_ids_in_sql = set( CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) else: raise NotImplementedError(doc_type) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only, ) if doc_type == "CommCareCase" and couch_missing_cases: has_diffs = True print( shell_red("%s cases could not be loaded from Couch" % couch_missing_cases)) if not short: for case_id in db.get_missing_doc_ids( "CommCareCase-couch"): print(case_id) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status(key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only) if diffs_only and not has_diffs: print( shell_green("No differences found between old and new docs!")) return has_diffs