def test_reprocess_unfinished_submission_case_create(self): case_id = uuid.uuid4().hex transaction_patch = patch( 'corehq.form_processor.backends.sql.processor.transaction') case_save_patch = patch( 'corehq.form_processor.backends.sql.dbaccessors.CaseAccessorSQL.save_case', side_effect=InternalError) with transaction_patch, case_save_patch, self.assertRaises( InternalError): self.factory.create_or_update_cases([ CaseStructure(case_id=case_id, attrs={ 'case_type': 'parent', 'create': True }) ]) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) # form that was saved before case error raised normal_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_state( self.domain, XFormInstanceSQL.NORMAL) self.assertEqual(1, len(normal_form_ids)) self.assertEqual(stubs[0].xform_id, normal_form_ids[0]) # shows error form (duplicate of form that was saved before case error) # this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed # the form didn't get saved # we don't really care about this form in this test error_forms = FormAccessorSQL.get_forms_by_type( self.domain, 'XFormError', 10) self.assertEqual(1, len(error_forms)) self.assertEqual(error_forms[0].orig_id, normal_form_ids[0]) self.assertEqual( 0, len(CaseAccessorSQL.get_case_ids_in_domain(self.domain))) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(0, len(result.ledgers)) case_ids = CaseAccessorSQL.get_case_ids_in_domain(self.domain) self.assertEqual(1, len(case_ids)) self.assertEqual(case_id, case_ids[0]) with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist): UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_get_case_ids_in_domain(self): case1 = _create_case(case_type='t1') case2 = _create_case(case_type='t1') case3 = _create_case(case_type='t2') case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN) self.assertEqual({case1.case_id, case2.case_id, case3.case_id}, set(case_ids)) case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN, 't1') self.assertEqual({case1.case_id, case2.case_id}, set(case_ids)) case2.domain = 'new_domain' CaseAccessorSQL.save_case(case2) case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN) self.assertEqual({case1.case_id, case3.case_id}, set(case_ids))
def test_get_deleted_case_ids(self): case1 = _create_case() case2 = _create_case() CaseAccessorSQL.soft_delete_cases(DOMAIN, [case1.case_id]) case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN) self.assertEqual(case_ids, [case2.case_id]) deleted = CaseAccessorSQL.get_deleted_case_ids_in_domain(DOMAIN) self.assertEquals(deleted, [case1.case_id])
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format(domain, status)) db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) form_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) ) form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status( "XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0)) has_diffs |= self._print_status( 'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) ) case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0)) has_diffs |= self._print_status( 'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status( key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only ) if diffs_only and not has_diffs: print(shell_green("No differences found between old and new docs!")) return has_diffs
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len( set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db()))) form_ids_in_sql = len( set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len( set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db()))) case_ids_in_sql = len( set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def check_for_sql_cases_without_existing_domain(): missing_domains_with_cases = set() for domain in set( _get_all_domains_that_have_ever_had_subscriptions()) - set( Domain.get_all_names()): if CaseAccessorSQL.get_case_ids_in_domain(domain): missing_domains_with_cases |= {domain} if missing_domains_with_cases: mail_admins_async.delay( 'There exist SQL cases belonging to a missing domain', six.text_type(missing_domains_with_cases)) elif _is_monday(): mail_admins_async.delay('All SQL cases belong to valid domains', '')
def delete_all_ledgers(domain): if should_use_sql_backend(domain): for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id) form_ids = {tx.form_id for tx in transactions} for form_id in form_ids: LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id) LedgerAccessorSQL.delete_ledger_values(case_id) else: from casexml.apps.stock.models import StockReport from casexml.apps.stock.models import StockTransaction stock_report_ids = StockReport.objects.filter(domain=domain).values_list('id', flat=True) StockReport.objects.filter(domain=domain).delete() StockTransaction.objects.filter(report_id__in=stock_report_ids).delete()
def check_for_sql_cases_without_existing_domain(): missing_domains_with_cases = set() for domain in set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names()): if CaseAccessorSQL.get_case_ids_in_domain(domain): missing_domains_with_cases |= {domain} if missing_domains_with_cases: mail_admins_async.delay( 'There exist SQL cases belonging to a missing domain', six.text_type(missing_domains_with_cases) ) elif _is_monday(): mail_admins_async.delay( 'All SQL cases belong to valid domains', '' )
def _blow_away_migration(domain): assert not should_use_sql_backend(domain) delete_diff_db(domain) for doc_type in doc_types(): sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type) FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False) sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain) FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False) sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain) CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids) sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain) CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def delete_all_v2_ledgers(domain=None): logger.debug("Deleting all V2 ledgers for domain %s", domain) def _delete_ledgers_for_case(case_id): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id) form_ids = {tx.form_id for tx in transactions} for form_id in form_ids: LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id) LedgerAccessorSQL.delete_ledger_values(case_id) if not domain: for ledger in iter_all_rows(LedgerReindexAccessor()): _delete_ledgers_for_case(ledger.case_id) else: for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain): _delete_ledgers_for_case(case_id)
def delete_all_v2_ledgers(domain=None): logger.debug("Deleting all V2 ledgers for domain %s", domain) def _delete_ledgers_for_case(case_id): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id) form_ids = {tx.form_id for tx in transactions} for form_id in form_ids: LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id) LedgerAccessorSQL.delete_ledger_values(case_id) if not domain: for db in get_sql_db_aliases_in_use(): for ledger in LedgerReindexAccessor().get_docs(db, None, limit=10000): _delete_ledgers_for_case(ledger.case_id) else: for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain): _delete_ledgers_for_case(case_id)
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) )) form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) )) case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def iter_all_changes(self, start_from=None): case_ids = CaseAccessorSQL.get_case_ids_in_domain(self.domain) for case in CaseAccessorSQL.get_cases(case_ids): yield _sql_case_to_change(case)
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format( domain, status)) db = open_state_db(domain, self.state_dir) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) if doc_type == "XFormInstance": form_ids_in_couch.update( get_doc_ids_in_domain_by_type(domain, "HQSubmission", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status(doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) form_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) ZERO = Counts(0, 0) if db.has_doc_counts(): doc_counts = db.get_doc_counts() couch_missing_cases = doc_counts.get("CommCareCase-couch", ZERO).missing else: doc_counts = None couch_missing_cases = 0 for doc_type in CASE_DOC_TYPES: if doc_counts is not None: counts = doc_counts.get(doc_type, ZERO) case_ids_in_couch = db.get_missing_doc_ids( doc_type) if counts.missing else set() case_ids_in_sql = counts elif doc_type == "CommCareCase": case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set( CaseAccessorSQL.get_case_ids_in_domain(domain)) elif doc_type == "CommCareCase-Deleted": case_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db())) case_ids_in_sql = set( CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) else: raise NotImplementedError(doc_type) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only, ) if doc_type == "CommCareCase" and couch_missing_cases: has_diffs = True print( shell_red("%s cases could not be loaded from Couch" % couch_missing_cases)) if not short: for case_id in db.get_missing_doc_ids( "CommCareCase-couch"): print(case_id) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status(key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only) if diffs_only and not has_diffs: print( shell_green("No differences found between old and new docs!")) return has_diffs