def handle(self, domain, **options): NotAllowed.check(domain) if not options['noinput']: confirm = input( """ Are you sure you want to hard delete all forms and cases in domain "{}"? This operation is PERMANENT. Type the domain's name again to continue, or anything else to cancel: """.format(domain) ) if confirm != domain: print("\n\t\tDomain deletion cancelled.") return logger.info('Hard deleting forms...') deleted_sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain) for form_id_chunk in chunked(with_progress_bar(deleted_sql_form_ids, stream=silence_during_tests()), 500): FormAccessorSQL.hard_delete_forms(domain, list(form_id_chunk), delete_attachments=True) logger.info('Hard deleting cases...') deleted_sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain) for case_id_chunk in chunked(with_progress_bar(deleted_sql_case_ids, stream=silence_during_tests()), 500): CaseAccessorSQL.hard_delete_cases(domain, list(case_id_chunk)) logger.info('Done.')
def test_hard_delete_forms_none_to_delete(self): for domain_name in [self.domain.name, self.domain2.name]: create_form_for_test(domain_name) create_form_for_test(domain_name, state=XFormInstanceSQL.ARCHIVED) self.assertEqual( len(FormAccessors(domain_name).get_all_form_ids_in_domain()), 1) self.domain.delete() self.assertEqual( len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0) self.assertEqual( len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1) self.assertEqual( len( FormAccessorSQL.get_deleted_form_ids_in_domain( self.domain.name)), 2) self.assertEqual( len( FormAccessorSQL.get_deleted_form_ids_in_domain( self.domain2.name)), 0) call_command('hard_delete_forms_and_cases_in_domain', self.domain2.name, noinput=True) self.assertEqual( len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0) self.assertEqual( len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1) self.assertEqual( len( FormAccessorSQL.get_deleted_form_ids_in_domain( self.domain.name)), 2) self.assertEqual( len( FormAccessorSQL.get_deleted_form_ids_in_domain( self.domain2.name)), 0)
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format(domain, status)) db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) form_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) ) form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status( "XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0)) has_diffs |= self._print_status( 'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) ) case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0)) has_diffs |= self._print_status( 'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status( key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only ) if diffs_only and not has_diffs: print(shell_green("No differences found between old and new docs!")) return has_diffs
def test_hard_delete_forms_none_to_delete(self): for domain_name in [self.domain.name, self.domain2.name]: create_form_for_test(domain_name) self.assertEqual(len(FormAccessors(domain_name).get_all_form_ids_in_domain()), 1) self.domain.delete() self.assertEqual(len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0) self.assertEqual(len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1) self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain.name)), 1) self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain2.name)), 0) call_command('hard_delete_forms_and_cases_in_domain', self.domain2.name, noinput=True) self.assertEqual(len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0) self.assertEqual(len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1) self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain.name)), 1) self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain2.name)), 0)
def test_deleted_form_migration(self): form = create_and_save_a_form(self.domain_name) FormAccessors(self.domain.name).soft_delete_forms( [form.form_id], datetime.utcnow(), 'test-deletion' ) self.assertEqual(1, len(get_doc_ids_in_domain_by_type( self.domain_name, "XFormInstance-Deleted", XFormInstance.get_db()) )) self._do_migration_and_assert_flags(self.domain_name) self.assertEqual(1, len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain_name))) self._compare_diffs([])
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len( set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db()))) form_ids_in_sql = len( set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len( set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db()))) case_ids_in_sql = len( set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def _blow_away_migration(domain): assert not should_use_sql_backend(domain) delete_diff_db(domain) for doc_type in doc_types(): sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type) FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False) sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain) FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False) sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain) CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids) sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain) CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def test_edited_deleted_form(self): form = create_and_save_a_form(self.domain_name) form.edited_on = datetime.utcnow() - timedelta(days=400) form.save() FormAccessors(self.domain.name).soft_delete_forms([form.form_id], datetime.utcnow(), 'test-deletion') self.assertEqual( get_doc_ids_in_domain_by_type(form.domain, "XFormInstance-Deleted", XFormInstance.get_db()), [form.form_id], ) self._do_migration_and_assert_flags(form.domain) self.assertEqual( FormAccessorSQL.get_deleted_form_ids_in_domain(form.domain), [form.form_id], ) self._compare_diffs([])
def test_edited_deleted_form(self): form = create_and_save_a_form(self.domain_name) form.edited_on = datetime.utcnow() - timedelta(days=400) form.save() FormAccessors(self.domain.name).soft_delete_forms( [form.form_id], datetime.utcnow(), 'test-deletion' ) self.assertEqual( get_doc_ids_in_domain_by_type( form.domain, "XFormInstance-Deleted", XFormInstance.get_db() ), [form.form_id], ) self._do_migration_and_assert_flags(form.domain) self.assertEqual( FormAccessorSQL.get_deleted_form_ids_in_domain(form.domain), [form.form_id], ) self._compare_diffs([])
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) )) form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) )) case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format( domain, status)) db = open_state_db(domain, self.state_dir) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) if doc_type == "XFormInstance": form_ids_in_couch.update( get_doc_ids_in_domain_by_type(domain, "HQSubmission", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status(doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) form_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) ZERO = Counts(0, 0) if db.has_doc_counts(): doc_counts = db.get_doc_counts() couch_missing_cases = doc_counts.get("CommCareCase-couch", ZERO).missing else: doc_counts = None couch_missing_cases = 0 for doc_type in CASE_DOC_TYPES: if doc_counts is not None: counts = doc_counts.get(doc_type, ZERO) case_ids_in_couch = db.get_missing_doc_ids( doc_type) if counts.missing else set() case_ids_in_sql = counts elif doc_type == "CommCareCase": case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set( CaseAccessorSQL.get_case_ids_in_domain(domain)) elif doc_type == "CommCareCase-Deleted": case_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db())) case_ids_in_sql = set( CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) else: raise NotImplementedError(doc_type) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only, ) if doc_type == "CommCareCase" and couch_missing_cases: has_diffs = True print( shell_red("%s cases could not be loaded from Couch" % couch_missing_cases)) if not short: for case_id in db.get_missing_doc_ids( "CommCareCase-couch"): print(case_id) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status(key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only) if diffs_only and not has_diffs: print( shell_green("No differences found between old and new docs!")) return has_diffs