def show_diffs(self, domain): db = get_diff_db(domain) diffs = sorted(db.get_diffs(), key=lambda d: d.kind) for doc_type, diffs in groupby(diffs, key=lambda d: d.kind): print('-' * 50, "Diffs for {}".format(doc_type), '-' * 50) for diff in diffs: print('[{}({})] {}'.format(doc_type, diff.doc_id, diff.json_diff))
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format(domain, status)) db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) form_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) ) form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status( "XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0)) has_diffs |= self._print_status( 'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) ) case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0)) has_diffs |= self._print_status( 'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status( key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only ) if diffs_only and not has_diffs: print(shell_green("No differences found between old and new docs!")) return has_diffs
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len( set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db()))) form_ids_in_sql = len( set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len( set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db()))) case_ids_in_sql = len( set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) )) form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) )) case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def show_diffs(self, domain): db = get_diff_db(domain) for diff in db.get_diffs(): print '[{}({})] {}'.format(diff.kind, diff.doc_id, diff.json_diff)
def _compare_diffs(self, expected): diffs = get_diff_db(self.domain_name).get_diffs() json_diffs = [(diff.kind, diff.json_diff) for diff in diffs] self.assertEqual(expected, json_diffs)