def test_reprocess_unfinished_submission_case_create(self):
case_id = uuid.uuid4().hex
transaction_patch = patch(
'corehq.form_processor.backends.sql.processor.transaction')
case_save_patch = patch(
'corehq.form_processor.backends.sql.dbaccessors.CaseAccessorSQL.save_case',
side_effect=InternalError)
with transaction_patch, case_save_patch, self.assertRaises(
InternalError):
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id,
attrs={
'case_type': 'parent',
'create': True
})
])
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
# form that was saved before case error raised
normal_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_state(
self.domain, XFormInstanceSQL.NORMAL)
self.assertEqual(1, len(normal_form_ids))
self.assertEqual(stubs[0].xform_id, normal_form_ids[0])
# shows error form (duplicate of form that was saved before case error)
# this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed
# the form didn't get saved
# we don't really care about this form in this test
error_forms = FormAccessorSQL.get_forms_by_type(
self.domain, 'XFormError', 10)
self.assertEqual(1, len(error_forms))
self.assertEqual(error_forms[0].orig_id, normal_form_ids[0])
self.assertEqual(
0, len(CaseAccessorSQL.get_case_ids_in_domain(self.domain)))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(0, len(result.ledgers))
case_ids = CaseAccessorSQL.get_case_ids_in_domain(self.domain)
self.assertEqual(1, len(case_ids))
self.assertEqual(case_id, case_ids[0])
with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist):
UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_get_case_ids_in_domain(self):
case1 = _create_case(case_type='t1')
case2 = _create_case(case_type='t1')
case3 = _create_case(case_type='t2')
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN)
self.assertEqual({case1.case_id, case2.case_id, case3.case_id}, set(case_ids))
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN, 't1')
self.assertEqual({case1.case_id, case2.case_id}, set(case_ids))
case2.domain = 'new_domain'
CaseAccessorSQL.save_case(case2)
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN)
self.assertEqual({case1.case_id, case3.case_id}, set(case_ids))
def test_get_case_ids_in_domain(self):
case1 = _create_case(case_type='t1')
case2 = _create_case(case_type='t1')
case3 = _create_case(case_type='t2')
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN)
self.assertEqual({case1.case_id, case2.case_id, case3.case_id}, set(case_ids))
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN, 't1')
self.assertEqual({case1.case_id, case2.case_id}, set(case_ids))
case2.domain = 'new_domain'
CaseAccessorSQL.save_case(case2)
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN)
self.assertEqual({case1.case_id, case3.case_id}, set(case_ids))
def test_get_deleted_case_ids(self):
case1 = _create_case()
case2 = _create_case()
CaseAccessorSQL.soft_delete_cases(DOMAIN, [case1.case_id])
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN)
self.assertEqual(case_ids, [case2.case_id])
deleted = CaseAccessorSQL.get_deleted_case_ids_in_domain(DOMAIN)
self.assertEquals(deleted, [case1.case_id])
def test_get_deleted_case_ids(self):
case1 = _create_case()
case2 = _create_case()
CaseAccessorSQL.soft_delete_cases(DOMAIN, [case1.case_id])
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN)
self.assertEqual(case_ids, [case2.case_id])
deleted = CaseAccessorSQL.get_deleted_case_ids_in_domain(DOMAIN)
self.assertEquals(deleted, [case1.case_id])
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def get_diff_stats(self, domain):
db = get_diff_db(domain)
diff_stats = db.get_diff_stats()
stats = {}
def _update_stats(doc_type, couch_count, sql_count):
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
if diff_count or couch_count != sql_count:
stats[doc_type] = (couch_count, sql_count, diff_count,
num_docs_with_diffs)
for doc_type in doc_types():
form_ids_in_couch = len(set(get_form_ids_by_type(domain,
doc_type)))
form_ids_in_sql = len(
set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type)))
_update_stats(doc_type, form_ids_in_couch, form_ids_in_sql)
form_ids_in_couch = len(
set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db())))
form_ids_in_sql = len(
set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)))
_update_stats("XFormInstance-Deleted", form_ids_in_couch,
form_ids_in_sql)
case_ids_in_couch = len(set(get_case_ids_in_domain(domain)))
case_ids_in_sql = len(
set(CaseAccessorSQL.get_case_ids_in_domain(domain)))
_update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql)
if self.strict:
# only care about these in strict mode
case_ids_in_couch = len(
set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db())))
case_ids_in_sql = len(
set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)))
_update_stats("CommCareCase-Deleted", case_ids_in_couch,
case_ids_in_sql)
if diff_stats:
for key in diff_stats.keys():
_update_stats(key, 0, 0)
return stats
def check_for_sql_cases_without_existing_domain():
missing_domains_with_cases = set()
for domain in set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names()):
if CaseAccessorSQL.get_case_ids_in_domain(domain):
missing_domains_with_cases |= {domain}
if missing_domains_with_cases:
mail_admins_async.delay(
'There exist SQL cases belonging to a missing domain',
six.text_type(missing_domains_with_cases))
elif _is_monday():
mail_admins_async.delay('All SQL cases belong to valid domains', '')
def delete_all_ledgers(domain):
if should_use_sql_backend(domain):
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
else:
from casexml.apps.stock.models import StockReport
from casexml.apps.stock.models import StockTransaction
stock_report_ids = StockReport.objects.filter(domain=domain).values_list('id', flat=True)
StockReport.objects.filter(domain=domain).delete()
StockTransaction.objects.filter(report_id__in=stock_report_ids).delete()
def check_for_sql_cases_without_existing_domain():
missing_domains_with_cases = set()
for domain in set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names()):
if CaseAccessorSQL.get_case_ids_in_domain(domain):
missing_domains_with_cases |= {domain}
if missing_domains_with_cases:
mail_admins_async.delay(
'There exist SQL cases belonging to a missing domain',
six.text_type(missing_domains_with_cases)
)
elif _is_monday():
mail_admins_async.delay(
'All SQL cases belong to valid domains', ''
)
def _blow_away_migration(domain):
assert not should_use_sql_backend(domain)
delete_diff_db(domain)
for doc_type in doc_types():
sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def _blow_away_migration(domain):
assert not should_use_sql_backend(domain)
delete_diff_db(domain)
for doc_type in doc_types():
sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for ledger in iter_all_rows(LedgerReindexAccessor()):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for ledger in iter_all_rows(LedgerReindexAccessor()):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for db in get_sql_db_aliases_in_use():
for ledger in LedgerReindexAccessor().get_docs(db, None, limit=10000):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for db in get_sql_db_aliases_in_use():
for ledger in LedgerReindexAccessor().get_docs(db, None, limit=10000):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def get_diff_stats(self, domain):
db = get_diff_db(domain)
diff_stats = db.get_diff_stats()
stats = {}
def _update_stats(doc_type, couch_count, sql_count):
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
if diff_count or couch_count != sql_count:
stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs)
for doc_type in doc_types():
form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type)))
form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)))
_update_stats(doc_type, form_ids_in_couch, form_ids_in_sql)
form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
))
form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)))
_update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql)
case_ids_in_couch = len(set(get_case_ids_in_domain(domain)))
case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain)))
_update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql)
if self.strict:
# only care about these in strict mode
case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
))
case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)))
_update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql)
if diff_stats:
for key in diff_stats.keys():
_update_stats(key, 0, 0)
return stats
def iter_all_changes(self, start_from=None):
case_ids = CaseAccessorSQL.get_case_ids_in_domain(self.domain)
for case in CaseAccessorSQL.get_cases(case_ids):
yield _sql_case_to_change(case)
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = open_state_db(domain, self.state_dir)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
if doc_type == "XFormInstance":
form_ids_in_couch.update(
get_doc_ids_in_domain_by_type(domain, "HQSubmission",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
ZERO = Counts(0, 0)
if db.has_doc_counts():
doc_counts = db.get_doc_counts()
couch_missing_cases = doc_counts.get("CommCareCase-couch",
ZERO).missing
else:
doc_counts = None
couch_missing_cases = 0
for doc_type in CASE_DOC_TYPES:
if doc_counts is not None:
counts = doc_counts.get(doc_type, ZERO)
case_ids_in_couch = db.get_missing_doc_ids(
doc_type) if counts.missing else set()
case_ids_in_sql = counts
elif doc_type == "CommCareCase":
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(
CaseAccessorSQL.get_case_ids_in_domain(domain))
elif doc_type == "CommCareCase-Deleted":
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
else:
raise NotImplementedError(doc_type)
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type,
case_ids_in_couch,
case_ids_in_sql,
diff_count,
num_docs_with_diffs,
short,
diffs_only,
)
if doc_type == "CommCareCase" and couch_missing_cases:
has_diffs = True
print(
shell_red("%s cases could not be loaded from Couch" %
couch_missing_cases))
if not short:
for case_id in db.get_missing_doc_ids(
"CommCareCase-couch"):
print(case_id)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
