def tearDownClass(cls):
CaseAccessorSQL.hard_delete_cases(
cls.domain,
cls.created_case_ids
)
cls.domain_obj.delete()
super(BaseICDSTest, cls).tearDownClass()
def test_deleted_case_migration(self):
parent_case_id = uuid.uuid4().hex
child_case_id = uuid.uuid4().hex
parent_case = create_and_save_a_case(self.domain_name, case_id=parent_case_id, case_name='test parent')
child_case = create_and_save_a_case(self.domain_name, case_id=child_case_id, case_name='test child')
set_parent_case(self.domain_name, child_case, parent_case)
form_ids = self._get_form_ids()
self.assertEqual(3, len(form_ids))
FormAccessors(self.domain.name).soft_delete_forms(
form_ids, datetime.utcnow(), 'test-deletion-with-cases'
)
CaseAccessors(self.domain.name).soft_delete_cases(
[parent_case_id, child_case_id], datetime.utcnow(), 'test-deletion-with-cases'
)
self.assertEqual(2, len(get_doc_ids_in_domain_by_type(
self.domain_name, "CommCareCase-Deleted", XFormInstance.get_db())
))
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(2, len(CaseAccessorSQL.get_deleted_case_ids_in_domain(self.domain_name)))
self._compare_diffs([])
parent_transactions = CaseAccessorSQL.get_transactions(parent_case_id)
self.assertEqual(2, len(parent_transactions))
self.assertTrue(parent_transactions[0].is_case_create)
self.assertTrue(parent_transactions[1].is_form_transaction)
child_transactions = CaseAccessorSQL.get_transactions(child_case_id)
self.assertEqual(2, len(child_transactions))
self.assertTrue(child_transactions[0].is_case_create)
self.assertTrue(child_transactions[1].is_case_index)
def _create_case_transactions(case):
case.track_create(CaseTransaction(
case=case,
form_id=uuid.uuid4().hex,
server_date=datetime.utcnow(),
type=CaseTransaction.TYPE_FORM,
revoked=False
))
# exclude revoked
case.track_create(CaseTransaction(
case=case,
form_id=uuid.uuid4().hex,
server_date=datetime.utcnow(),
type=CaseTransaction.TYPE_FORM,
revoked=True
))
# exclude based on type
case.track_create(CaseTransaction(
case=case,
form_id=uuid.uuid4().hex,
server_date=datetime.utcnow(),
type=CaseTransaction.TYPE_REBUILD_FORM_ARCHIVED,
revoked=False
))
form_ids = [t.form_id for t in case.get_tracked_models_to_create(CaseTransaction)]
CaseAccessorSQL.save_case(case)
return form_ids
def test_get_attachments(self):
case = _create_case()
case.track_create(CaseAttachmentSQL(
case=case,
attachment_id=uuid.uuid4().hex,
name='pic.jpg',
content_type='image/jpeg',
blob_id='125',
identifier='pic1',
md5='123',
))
case.track_create(CaseAttachmentSQL(
case=case,
attachment_id=uuid.uuid4().hex,
name='doc',
content_type='text/xml',
blob_id='126',
identifier='doc1',
md5='123',
))
CaseAccessorSQL.save_case(case)
with self.assertNumQueries(1, using=db_for_read_write(CaseAttachmentSQL)):
attachments = CaseAccessorSQL.get_attachments(case.case_id)
self.assertEqual(2, len(attachments))
sorted_attachments = sorted(attachments, key=lambda x: x.name)
for att in attachments:
self.assertEqual(case.case_id, att.case_id)
self.assertEqual('doc', sorted_attachments[0].name)
self.assertEqual('pic.jpg', sorted_attachments[1].name)
def test_archive_unarchive_form(self):
case_id = uuid.uuid4().hex
form = create_form_for_test(DOMAIN, case_id=case_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
self.assertEqual(0, len(form.history))
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
FormAccessorSQL.archive_form(form, 'user1')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ARCHIVED, form.state)
operations = form.history
self.assertEqual(1, len(operations))
self.assertEqual(form.form_id, operations[0].form_id)
self.assertEqual('user1', operations[0].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertTrue(transactions[0].revoked)
FormAccessorSQL.unarchive_form(form, 'user2')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
operations = form.history
self.assertEqual(2, len(operations))
self.assertEqual(form.form_id, operations[1].form_id)
self.assertEqual('user2', operations[1].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
def test_get_attachment_by_name(self):
case = _create_case()
case.track_create(CaseAttachmentSQL(
case=case,
attachment_id=uuid.uuid4().hex,
name='pic.jpg',
content_type='image/jpeg',
blob_id='123',
identifier='pic1',
md5='123'
))
case.track_create(CaseAttachmentSQL(
case=case,
attachment_id=uuid.uuid4().hex,
name='my_doc',
content_type='text/xml',
blob_id='124',
identifier='doc1',
md5='123'
))
CaseAccessorSQL.save_case(case)
with self.assertRaises(AttachmentNotFound):
CaseAccessorSQL.get_attachment_by_identifier(case.case_id, 'missing')
with self.assertNumQueries(1, using=db_for_read_write(CaseAttachmentSQL)):
attachment_meta = CaseAccessorSQL.get_attachment_by_identifier(case.case_id, 'pic1')
self.assertEqual(case.case_id, attachment_meta.case_id)
self.assertEqual('pic.jpg', attachment_meta.name)
self.assertEqual('image/jpeg', attachment_meta.content_type)
def test_get_extension_case_ids(self):
# Create case and index
referenced_id = uuid.uuid4().hex
case = _create_case()
extension_index = CommCareCaseIndexSQL(
case=case,
identifier="task",
referenced_type="task",
referenced_id=referenced_id,
relationship_id=CommCareCaseIndexSQL.EXTENSION
)
case.track_create(extension_index)
CaseAccessorSQL.save_case(case)
# Create irrelevant case
other_case = _create_case()
child_index = CommCareCaseIndexSQL(
case=other_case,
identifier='parent',
referenced_type='mother',
referenced_id=referenced_id,
relationship_id=CommCareCaseIndexSQL.CHILD
)
case.track_create(child_index)
CaseAccessorSQL.save_case(other_case)
self.assertEqual(
CaseAccessorSQL.get_extension_case_ids(DOMAIN, [referenced_id]),
[case.case_id]
)
def hard_rebuild_case(domain, case_id, detail, lock=True):
if lock:
# only record metric if locking since otherwise it has been
# (most likley) recorded elsewhere
case_load_counter("rebuild_case", domain)()
case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock)
found = bool(case)
if not found:
case = CommCareCaseSQL(case_id=case_id, domain=domain)
if lock:
lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id)
acquire_lock(lock_obj, degrade_gracefully=False)
try:
assert case.domain == domain, (case.domain, domain)
case, rebuild_transaction = FormProcessorSQL._rebuild_case_from_transactions(case, detail)
if case.is_deleted and not case.is_saved():
return None
case.server_modified_on = rebuild_transaction.server_date
CaseAccessorSQL.save_case(case)
publish_case_saved(case)
return case
finally:
release_lock(lock_obj, degrade_gracefully=True)
def test_reconcile_transactions_within_fudge_factor(self, soft_assert_mock):
""" tests a transanction with an early client date and late server date """
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-11 06:00"):
new_old_xform = self._create_form()
with freeze_time("2018-10-10 18:00"):
new_old_trans = self._create_case_transaction(case, new_old_xform)
with freeze_time("2018-10-11 06:00"):
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
with freeze_time("2018-10-11"):
new_old_xform = self._create_form()
new_old_trans = self._create_case_transaction(case, new_old_xform)
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertTrue(update_strategy.reconcile_transactions_if_necessary())
self._check_for_reconciliation_error_soft_assert(soft_assert_mock)
CaseAccessorSQL.save_case(case)
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertFalse(update_strategy.reconcile_transactions_if_necessary())
self._check_for_reconciliation_error_soft_assert(soft_assert_mock)
def _create_ledger(domain, entry_id, balance, case_id=None, section_id='stock'):
user_id = 'user1'
utcnow = datetime.utcnow()
case_id = case_id or uuid.uuid4().hex
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type='',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
)
CaseAccessorSQL.save_case(case)
ledger = LedgerValue(
domain=domain,
case_id=case_id,
section_id=section_id,
entry_id=entry_id,
balance=balance,
last_modified=utcnow
)
LedgerAccessorSQL.save_ledger_values([ledger])
return ledger
def test_get_attachments(self):
case = _create_case()
case.track_create(CaseAttachmentSQL(
case=case,
attachment_id=uuid.uuid4().hex,
name='pic.jpg',
content_type='image/jpeg'
))
case.track_create(CaseAttachmentSQL(
case=case,
attachment_id=uuid.uuid4().hex,
name='doc',
content_type='text/xml'
))
CaseAccessorSQL.save_case(case)
with self.assertRaises(AttachmentNotFound):
CaseAccessorSQL.get_attachment_by_name(case.case_id, 'missing')
with self.assertNumQueries(1, using=db_for_read_write(CaseAttachmentSQL)):
attachments = CaseAccessorSQL.get_attachments(case.case_id)
self.assertEqual(2, len(attachments))
sorted_attachments = sorted(attachments, key=lambda x: x.name)
for att in attachments:
self.assertEqual(case.case_id, att.case_id)
self.assertEqual('doc', sorted_attachments[0].name)
self.assertEqual('pic.jpg', sorted_attachments[1].name)
def test_edit_form_that_removes_ledgers(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
form_id = uuid.uuid4().hex
submit_case_blocks([
get_single_balance_block(self.case.case_id, self.product_a._id, 100)],
DOMAIN,
form_id=form_id
)
self._assert_ledger_state(100)
transactions = CaseAccessorSQL.get_transactions(self.case.case_id)
self.assertEqual(2, len(transactions))
self.assertTrue(transactions[0].is_form_transaction)
self.assertTrue(transactions[1].is_form_transaction)
self.assertTrue(transactions[1].is_ledger_transaction)
submit_case_blocks([
CaseBlock(case_id=self.case.case_id).as_string().decode('utf-8')],
DOMAIN,
form_id=form_id
)
self._assert_ledger_state(0)
transactions = CaseAccessorSQL.get_transactions(self.case.case_id)
self.assertEqual(3, len(transactions))
self.assertTrue(transactions[0].is_form_transaction)
# ordering not guaranteed since they have the same date
self.assertTrue(transactions[1].is_form_transaction)
self.assertFalse(transactions[1].is_ledger_transaction) # no longer a ledger transaction
self.assertTrue(transactions[2].is_case_rebuild)
self._assert_transactions([])
def test_get_case_by_location(self):
case = _create_case(case_type=SupplyPointCaseMixin.CASE_TYPE)
location_id = uuid.uuid4().hex
case.location_id = location_id
CaseAccessorSQL.save_case(case)
fetched_case = CaseAccessorSQL.get_case_by_location(DOMAIN, location_id)
self.assertEqual(case.id, fetched_case.id)
def test_get_closed_case_ids(self):
case1 = _create_case(user_id="user1")
case2 = _create_case(user_id="user1")
case3 = _create_case(user_id="user2")
case2.closed = True
CaseAccessorSQL.save_case(case2)
self.assertEqual(CaseAccessorSQL.get_closed_case_ids_for_owner(DOMAIN, "user1"), [case2.case_id])
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
def test_case_has_transactions_since_sync(self):
case1 = _create_case()
_create_case_transactions(case1)
self.assertTrue(
CaseAccessorSQL.case_has_transactions_since_sync(case1.case_id, "foo", datetime(1992, 01, 30))
)
self.assertFalse(
CaseAccessorSQL.case_has_transactions_since_sync(case1.case_id, "foo", datetime.utcnow())
)
def test_get_deleted_case_ids_by_owner(self):
user_id = uuid.uuid4().hex
case1 = _create_case(user_id=user_id)
case2 = _create_case(user_id=user_id)
case3 = _create_case(user_id=user_id)
CaseAccessorSQL.soft_delete_cases(DOMAIN, [case1.case_id, case2.case_id])
case_ids = CaseAccessorSQL.get_deleted_case_ids_by_owner(DOMAIN, user_id)
self.assertEqual(set(case_ids), {case1.case_id, case2.case_id})
def test_get_deleted_case_ids(self):
case1 = _create_case()
case2 = _create_case()
CaseAccessorSQL.soft_delete_cases(DOMAIN, [case1.case_id])
case_ids = CaseAccessorSQL.get_case_ids_in_domain(DOMAIN)
self.assertEqual(case_ids, [case2.case_id])
deleted = CaseAccessorSQL.get_deleted_case_ids_in_domain(DOMAIN)
self.assertEquals(deleted, [case1.case_id])
def create_case(domain, case_type, **kwargs):
case = CaseFactory(domain).create_case(case_type=case_type, **kwargs)
try:
yield case
finally:
if should_use_sql_backend(domain):
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
else:
case.delete()
def test_get_transaction_by_form_id(self):
form_id = uuid.uuid4().hex
case = _create_case(form_id=form_id)
transaction = CaseAccessorSQL.get_transaction_by_form_id(case.case_id, form_id)
self.assertEqual(form_id, transaction.form_id)
self.assertEqual(case.case_id, transaction.case_id)
transaction = CaseAccessorSQL.get_transaction_by_form_id(case.case_id, 'wrong')
self.assertIsNone(transaction)
def test_hard_delete_case(self):
case1 = _create_case()
case2 = _create_case(domain='other_domain')
self.addCleanup(lambda: CaseAccessorSQL.hard_delete_cases('other_domain', [case2.case_id]))
case1.track_create(CommCareCaseIndexSQL(
case=case1,
identifier='parent',
referenced_type='mother',
referenced_id=uuid.uuid4().hex,
relationship_id=CommCareCaseIndexSQL.CHILD
))
case1.track_create(CaseAttachmentSQL(
case=case1,
attachment_id=uuid.uuid4().hex,
name='pic.jpg',
content_type='image/jpeg',
blob_id='122',
md5='123',
identifier='pic.jpg',
))
CaseAccessorSQL.save_case(case1)
num_deleted = CaseAccessorSQL.hard_delete_cases(DOMAIN, [case1.case_id, case2.case_id])
self.assertEqual(1, num_deleted)
with self.assertRaises(CaseNotFound):
CaseAccessorSQL.get_case(case1.case_id)
self.assertEqual([], CaseAccessorSQL.get_indices(case1.domain, case1.case_id))
self.assertEqual([], CaseAccessorSQL.get_attachments(case1.case_id))
self.assertEqual([], CaseAccessorSQL.get_transactions(case1.case_id))
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def test_save_case_update_transaction(self):
case = _create_case()
[transaction] = CaseAccessorSQL.get_transactions(case.case_id)
transaction.revoked = True
# hack to call the sql function with an already saved transaction
case.track_create(transaction)
with self.assertRaises(CaseSaveError):
CaseAccessorSQL.save_case(case)
def _update_case(domain, case_id, server_modified_on, last_visit_date=None):
accessors = CaseAccessors(domain)
case = accessors.get_case(case_id)
case.server_modified_on = server_modified_on
if last_visit_date:
set_case_property_directly(case, 'last_visit_date', last_visit_date.strftime('%Y-%m-%d'))
if should_use_sql_backend(domain):
CaseAccessorSQL.save_case(case)
else:
# can't call case.save() since it overrides the server_modified_on property
CommCareCase.get_db().save_doc(case.to_json())
def test_get_case_owner_ids(self):
_create_case(user_id='user1', case_id='123') # get's sharded to p1
_create_case(user_id='user2', case_id='125') # get's sharded to p2
_create_case(user_id='user1')
_create_case(domain='other_domain', user_id='user3')
owners = CaseAccessorSQL.get_case_owner_ids('other_domain')
self.assertEqual({'user3'}, owners)
owners = CaseAccessorSQL.get_case_owner_ids(DOMAIN)
self.assertEqual({'user1', 'user2'}, owners)
def test_get_all_reverse_indices_info(self):
# Create case and indexes
case = _create_case()
referenced_id1 = uuid.uuid4().hex
referenced_id2 = uuid.uuid4().hex
extension_index = CommCareCaseIndexSQL(
case=case,
identifier="task",
referenced_type="task",
referenced_id=referenced_id1,
relationship_id=CommCareCaseIndexSQL.EXTENSION
)
case.track_create(extension_index)
child_index = CommCareCaseIndexSQL(
case=case,
identifier='parent',
referenced_type='mother',
referenced_id=referenced_id2,
relationship_id=CommCareCaseIndexSQL.CHILD
)
case.track_create(child_index)
CaseAccessorSQL.save_case(case)
# Create irrelevant case and index
other_case = _create_case()
other_child_index = CommCareCaseIndexSQL(
case=other_case,
identifier='parent',
referenced_type='mother',
referenced_id=case.case_id,
relationship_id=CommCareCaseIndexSQL.CHILD
)
other_case.track_create(other_child_index)
CaseAccessorSQL.save_case(other_case)
self.assertEqual(
set(CaseAccessorSQL.get_all_reverse_indices_info(DOMAIN, [referenced_id1, referenced_id2])),
{
CaseIndexInfo(
case_id=case.case_id,
identifier=u'task',
referenced_id=referenced_id1,
referenced_type=u'task',
relationship=CommCareCaseIndexSQL.EXTENSION,
),
CaseIndexInfo(
case_id=case.case_id,
identifier=u'parent',
referenced_id=referenced_id2,
referenced_type=u'mother',
relationship=CommCareCaseIndexSQL.CHILD
),
}
)
def test_get_transactions(self):
form_id = uuid.uuid4().hex
case = _create_case(form_id=form_id)
transactions = CaseAccessorSQL.get_transactions(case.case_id)
self.assertEqual(1, len(transactions))
self.assertEqual(form_id, transactions[0].form_id)
form_ids = _create_case_transactions(case)
transactions = CaseAccessorSQL.get_transactions(case.case_id)
self.assertEqual(6, len(transactions))
self.assertEqual([form_id] + form_ids, [t.form_id for t in transactions])
def get_case_with_lock(case_id, lock=False, strip_history=False, wrap=False):
try:
if lock:
try:
return CommCareCaseSQL.get_locked_obj(_id=case_id)
except redis.RedisError:
case = CaseAccessorSQL.get_case(case_id)
else:
case = CaseAccessorSQL.get_case(case_id)
except CaseNotFound:
return None, None
return case, None
def hard_rebuild_case(domain, case_id, detail):
try:
case = CaseAccessorSQL.get_case(case_id)
assert case.domain == domain
found = True
except CaseNotFound:
case = CommCareCaseSQL(case_id=case_id, domain=domain)
found = False
case = FormProcessorSQL._rebuild_case_from_transactions(case, detail)
if case.is_deleted and not found:
return None
CaseAccessorSQL.save_case(case)
def _create_case_with_index(referenced_case_id):
case = _create_case()
index1 = CommCareCaseIndexSQL(
case=case,
identifier='parent',
referenced_type='mother',
referenced_id=referenced_case_id,
relationship_id=CommCareCaseIndexSQL.CHILD
)
case.track_create(index1)
CaseAccessorSQL.save_case(case)
return case.case_id
def test_get_reverse_indexed_cases(self):
referenced_case_ids = [uuid.uuid4().hex, uuid.uuid4().hex]
_create_case_with_index(uuid.uuid4().hex, case_is_deleted=True) # case shouldn't be included in results
expected_case_ids = [
_create_case_with_index(referenced_case_ids[0], case_type='bambino')[0].case_id,
_create_case_with_index(referenced_case_ids[1], case_type='child')[0].case_id,
]
cases = CaseAccessorSQL.get_reverse_indexed_cases(DOMAIN, referenced_case_ids)
self.assertEqual(2, len(cases))
self.assertEqual(set(expected_case_ids), {c.case_id for c in cases})
cases = CaseAccessorSQL.get_reverse_indexed_cases(
DOMAIN, referenced_case_ids, case_types=['child'], is_closed=False)
self.assertEqual(1, len(cases))
cases[0].closed = True
CaseAccessorSQL.save_case(cases[0])
cases = CaseAccessorSQL.get_reverse_indexed_cases(DOMAIN, referenced_case_ids, is_closed=True)
self.assertEqual(1, len(cases))
def _transactions_by_type(self, transaction_type):
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL
if self.is_saved():
transactions = CaseAccessorSQL.get_transactions_by_type(self.case_id, transaction_type)
else:
transactions = []
transactions += filter(
lambda t: (t.type & transaction_type) == transaction_type,
self.get_tracked_models_to_create(CaseTransaction)
)
return transactions
def test_hard_delete_cases_none_to_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
CaseFactory(domain_name).create_case()
self.assertEqual(len(CaseAccessors(domain_name).get_case_ids_in_domain()), 1)
self.domain.delete()
self.assertEqual(len(CaseAccessors(self.domain.name).get_case_ids_in_domain()), 0)
self.assertEqual(len(CaseAccessors(self.domain2.name).get_case_ids_in_domain()), 1)
self.assertEqual(len(CaseAccessorSQL.get_deleted_case_ids_in_domain(self.domain.name)), 1)
self.assertEqual(len(CaseAccessorSQL.get_deleted_case_ids_in_domain(self.domain2.name)), 0)
call_command('hard_delete_forms_and_cases_in_domain', self.domain2.name, noinput=True)
self.assertEqual(len(CaseAccessors(self.domain.name).get_case_ids_in_domain()), 0)
self.assertEqual(len(CaseAccessors(self.domain2.name).get_case_ids_in_domain()), 1)
self.assertEqual(len(CaseAccessorSQL.get_deleted_case_ids_in_domain(self.domain.name)), 1)
self.assertEqual(len(CaseAccessorSQL.get_deleted_case_ids_in_domain(self.domain2.name)), 0)
def _copy_unprocessed_case(self, change):
couch_case = CommCareCase.wrap(change.get_document())
self.log_debug('Processing doc: {}({})'.format(couch_case['doc_type'], change.id))
try:
first_action = couch_case.actions[0]
except IndexError:
first_action = CommCareCaseAction()
sql_case = CommCareCaseSQL(
case_id=couch_case.case_id,
domain=self.domain,
type=couch_case.type or '',
name=couch_case.name,
owner_id=couch_case.owner_id or couch_case.user_id or '',
opened_on=couch_case.opened_on or first_action.date,
opened_by=couch_case.opened_by or first_action.user_id,
modified_on=couch_case.modified_on,
modified_by=couch_case.modified_by or couch_case.user_id or '',
server_modified_on=couch_case.server_modified_on,
closed=couch_case.closed,
closed_on=couch_case.closed_on,
closed_by=couch_case.closed_by,
deleted=True,
deletion_id=couch_case.deletion_id,
deleted_on=couch_case.deletion_date,
external_id=couch_case.external_id,
case_json=couch_case.dynamic_case_properties()
)
_migrate_case_actions(couch_case, sql_case)
_migrate_case_indices(couch_case, sql_case)
_migrate_case_attachments(couch_case, sql_case)
try:
CaseAccessorSQL.save_case(sql_case)
except IntegrityError:
# case re-created by form processing so just mark the case as deleted
CaseAccessorSQL.soft_delete_cases(
self.domain,
[sql_case.case_id],
sql_case.deleted_on,
sql_case.deletion_id
)
def _assert_case_revision(self,
rev_number,
last_modified,
expect_modified=False):
if should_use_sql_backend(self.domain):
modified_on = CaseAccessorSQL.get_last_modified_dates(
self.domain, [self.case_id])[self.case_id]
has_been_modified = modified_on != last_modified
self.assertEqual(expect_modified, has_been_modified)
else:
doc = self._get_case()
self.assertTrue(doc['_rev'].startswith('%s-' % rev_number))
def test_hard_delete_case(self):
case1 = _create_case()
case2 = _create_case(domain='other_domain')
self.addCleanup(lambda: CaseAccessorSQL.hard_delete_cases('other_domain', [case2.case_id]))
case1.track_create(CommCareCaseIndexSQL(
case=case1,
identifier='parent',
referenced_type='mother',
referenced_id=uuid.uuid4().hex,
relationship_id=CommCareCaseIndexSQL.CHILD
))
def get_case(self, case_id):
try:
return CaseAccessorSQL.get_case(case_id)
except CaseNotFound:
pass
try:
return CaseAccessorCouch.get_case(case_id)
except ResourceNotFound:
pass
return None
def get_transaction_by_form_id(self, form_id):
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL
transactions = filter(
lambda t: t.form_id == form_id,
self.get_tracked_models_to_create(CaseTransaction)
)
assert len(transactions) <= 1
transaction = transactions[0] if transactions else None
if not transaction:
transaction = CaseAccessorSQL.get_transaction_by_form_id(self.case_id, form_id)
return transaction
def test_get_open_case_ids_in_domain_by_type(self):
case1 = _create_case(user_id="user1", case_type='t1')
case2 = _create_case(user_id="user1", case_type='t1')
_create_case(user_id="user1", case_type='t1', closed=True)
_create_case(user_id="user2", case_type='t1')
_create_case(user_id="user1", case_type='t2')
case_ids = CaseAccessorSQL.get_open_case_ids_in_domain_by_type(DOMAIN, 't1', ["user1"])
self.assertEqual(
set(case_ids),
{case1.case_id, case2.case_id}
)
def _assert_case_revision(self,
rev_number,
last_modified,
expect_modified=False):
if should_use_sql_backend(self.domain):
self.assertEqual(
expect_modified,
CaseAccessorSQL.case_modified_since(self.case_id,
last_modified))
else:
doc = self._get_case()
self.assertTrue(doc['_rev'].startswith('%s-' % rev_number))
def _copy_unprocessed_cases(self):
doc_types = ['CommCareCase-Deleted']
changes = _get_case_iterator(self.domain,
doc_types=doc_types).iter_all_changes()
for change in self._with_progress(doc_types, changes):
couch_case = CommCareCase.wrap(change.get_document())
self.log_debug('Processing doc: {}({})'.format(
couch_case['doc_type'], change.id))
try:
first_action = couch_case.actions[0]
except IndexError:
first_action = CommCareCaseAction()
sql_case = CommCareCaseSQL(
case_id=couch_case.case_id,
domain=self.domain,
type=couch_case.type or '',
name=couch_case.name,
owner_id=couch_case.owner_id or couch_case.user_id or '',
opened_on=couch_case.opened_on or first_action.date,
opened_by=couch_case.opened_by or first_action.user_id,
modified_on=couch_case.modified_on,
modified_by=couch_case.modified_by or couch_case.user_id or '',
server_modified_on=couch_case.server_modified_on,
closed=couch_case.closed,
closed_on=couch_case.closed_on,
closed_by=couch_case.closed_by,
deleted=True,
deletion_id=couch_case.deletion_id,
deleted_on=couch_case.deletion_date,
external_id=couch_case.external_id,
case_json=couch_case.dynamic_case_properties())
_migrate_case_actions(couch_case, sql_case)
_migrate_case_indices(couch_case, sql_case)
_migrate_case_attachments(couch_case, sql_case)
try:
CaseAccessorSQL.save_case(sql_case)
except IntegrityError as e:
self.log_error("Unable to migrate case:\n{}\n{}".format(
couch_case.case_id, e))
def _copy_unprocessed_case(self, doc):
couch_case = CommCareCase.wrap(doc)
log.debug('Processing doc: %(doc_type)s(%(_id)s)', doc)
try:
first_action = couch_case.actions[0]
except IndexError:
first_action = CommCareCaseAction()
opened_on = couch_case.opened_on or first_action.date
sql_case = CommCareCaseSQL(
case_id=couch_case.case_id,
domain=self.domain,
type=couch_case.type or '',
name=couch_case.name,
owner_id=couch_case.owner_id or couch_case.user_id or '',
opened_on=opened_on,
opened_by=couch_case.opened_by or first_action.user_id,
modified_on=couch_case.modified_on or opened_on,
modified_by=couch_case.modified_by or couch_case.user_id or '',
server_modified_on=couch_case.server_modified_on,
closed=couch_case.closed,
closed_on=couch_case.closed_on,
closed_by=couch_case.closed_by,
deleted=True,
deletion_id=couch_case.deletion_id,
deleted_on=couch_case.deletion_date,
external_id=couch_case.external_id,
case_json=couch_case.dynamic_case_properties())
_migrate_case_actions(couch_case, sql_case)
_migrate_case_indices(couch_case, sql_case)
_migrate_case_attachments(couch_case, sql_case)
try:
CaseAccessorSQL.save_case(sql_case)
except IntegrityError:
# case re-created by form processing so just mark the case as deleted
CaseAccessorSQL.soft_delete_cases(self.domain, [sql_case.case_id],
sql_case.deleted_on,
sql_case.deletion_id)
finally:
self.case_diff_queue.enqueue(couch_case.case_id)
def test_deleted_case_migration(self):
parent_case_id = uuid.uuid4().hex
child_case_id = uuid.uuid4().hex
parent_case = create_and_save_a_case(self.domain_name,
case_id=parent_case_id,
case_name='test parent')
child_case = create_and_save_a_case(self.domain_name,
case_id=child_case_id,
case_name='test child')
set_parent_case(self.domain_name, child_case, parent_case)
form_ids = self._get_form_ids()
self.assertEqual(3, len(form_ids))
FormAccessors(self.domain.name).soft_delete_forms(
form_ids, datetime.utcnow(), 'test-deletion-with-cases')
CaseAccessors(self.domain.name).soft_delete_cases(
[parent_case_id, child_case_id], datetime.utcnow(),
'test-deletion-with-cases')
self.assertEqual(
2,
len(
get_doc_ids_in_domain_by_type(self.domain_name,
"CommCareCase-Deleted",
XFormInstance.get_db())))
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(
2,
len(
CaseAccessorSQL.get_deleted_case_ids_in_domain(
self.domain_name)))
self._compare_diffs([])
parent_transactions = CaseAccessorSQL.get_transactions(parent_case_id)
self.assertEqual(2, len(parent_transactions))
self.assertTrue(parent_transactions[0].is_case_create)
self.assertTrue(parent_transactions[1].is_form_transaction)
child_transactions = CaseAccessorSQL.get_transactions(child_case_id)
self.assertEqual(2, len(child_transactions))
self.assertTrue(child_transactions[0].is_case_create)
self.assertTrue(child_transactions[1].is_case_index)
def test_get_reverse_indexed_cases(self):
referenced_case_ids = [uuid.uuid4().hex, uuid.uuid4().hex]
_create_case_with_index(
referenced_case_ids[0],
case_is_deleted=True) # case shouldn't be included in results
expected_case_ids = [
_create_case_with_index(case_id)[0].case_id
for case_id in referenced_case_ids
]
cases = CaseAccessorSQL.get_reverse_indexed_cases(
DOMAIN, referenced_case_ids)
self.assertEqual(2, len(cases))
self.assertEqual(set(expected_case_ids), {c.case_id for c in cases})
def get_diff_stats(self, domain):
db = get_diff_db(domain)
diff_stats = db.get_diff_stats()
stats = {}
def _update_stats(doc_type, couch_count, sql_count):
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
if diff_count or couch_count != sql_count:
stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs)
for doc_type in doc_types():
form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type)))
form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)))
_update_stats(doc_type, form_ids_in_couch, form_ids_in_sql)
form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
))
form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)))
_update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql)
case_ids_in_couch = len(set(get_case_ids_in_domain(domain)))
case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain)))
_update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql)
if self.strict:
# only care about these in strict mode
case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
))
case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)))
_update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql)
if diff_stats:
for key in diff_stats.keys():
_update_stats(key, 0, 0)
return stats
def check_for_sql_cases_without_existing_domain():
missing_domains_with_cases = set()
for domain in set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names()):
if CaseAccessorSQL.get_case_ids_in_domain(domain):
missing_domains_with_cases |= {domain}
if missing_domains_with_cases:
mail_admins_async.delay(
'There exist SQL cases belonging to a missing domain',
six.text_type(missing_domains_with_cases))
elif _is_monday():
mail_admins_async.delay('All SQL cases belong to valid domains', '')
def test_get_case_by_external_id(self):
case1 = _create_case(domain=DOMAIN)
case1.external_id = '123'
CaseAccessorSQL.save_case(case1)
case2 = _create_case(domain='d2', case_type='t1')
case2.external_id = '123'
CaseAccessorSQL.save_case(case2)
self.addCleanup(lambda: CaseAccessorSQL.delete_all_cases('d2'))
[case] = CaseAccessorSQL.get_cases_by_external_id(DOMAIN, '123')
self.assertEqual(case.case_id, case1.case_id)
[case] = CaseAccessorSQL.get_cases_by_external_id('d2', '123')
self.assertEqual(case.case_id, case2.case_id)
self.assertEqual([], CaseAccessorSQL.get_cases_by_external_id('d2', '123', case_type='t2'))
def test_archive_unarchive_form(self):
case_id = uuid.uuid4().hex
form = create_form_for_test(DOMAIN, case_id=case_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
self.assertEqual(0, len(form.history))
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
# archive twice to check that it's idempotent
for i in range(2):
self.archive_form(form, 'user1')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ARCHIVED, form.state)
operations = form.history
self.assertEqual(i + 1, len(operations))
self.assertEqual(form.form_id, operations[i].form_id)
self.assertEqual('user1', operations[i].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions), transactions)
self.assertTrue(transactions[0].revoked)
# unarchive twice to check that it's idempotent
for i in range(2, 4):
self.unarchive_form(form, 'user2')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
operations = form.history
self.assertEqual(i + 1, len(operations))
self.assertEqual(form.form_id, operations[i].form_id)
self.assertEqual('user2', operations[i].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
def test_case_with_indices_migration(self):
parent_case_id = uuid.uuid4().hex
child_case_id = uuid.uuid4().hex
parent_case = create_and_save_a_case(self.domain_name, case_id=parent_case_id, case_name='test parent')
child_case = create_and_save_a_case(self.domain_name, case_id=child_case_id, case_name='test child')
set_parent_case(self.domain_name, child_case, parent_case)
self.assertEqual(2, len(self._get_case_ids()))
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(2, len(self._get_case_ids()))
self._compare_diffs([])
indices = CaseAccessorSQL.get_indices(self.domain_name, child_case_id)
self.assertEqual(1, len(indices))
self.assertEqual(parent_case_id, indices[0].referenced_id)
def test_get_extension_case_ids(self):
# Create case and index
referenced_id = uuid.uuid4().hex
case, _ = _create_case_with_index(referenced_id, identifier='task', referenced_type='task',
relationship_id=CommCareCaseIndexSQL.EXTENSION)
# Create irrelevant cases
_create_case_with_index(referenced_id)
_create_case_with_index(referenced_id, identifier='task', referenced_type='task',
relationship_id=CommCareCaseIndexSQL.EXTENSION, case_is_deleted=True)
self.assertEqual(
CaseAccessorSQL.get_extension_case_ids(DOMAIN, [referenced_id]),
[case.case_id]
)
def _diff_cases(self, couch_cases):
from corehq.apps.tzmigration.timezonemigration import json_diff
self.log_debug('Calculating case diffs for {} cases'.format(len(couch_cases)))
case_ids = list(couch_cases)
sql_cases = CaseAccessorSQL.get_cases(case_ids)
for sql_case in sql_cases:
couch_case = couch_cases[sql_case.case_id]
sql_case_json = sql_case.to_json()
diffs = json_diff(couch_case, sql_case_json, track_list_indices=False)
self.diff_db.add_diffs(
couch_case['doc_type'], sql_case.case_id,
filter_case_diffs(couch_case, sql_case_json, diffs, self.forms_that_touch_cases_without_actions)
)
self._diff_ledgers(case_ids)
def handle(self, domain, **options):
domain = options.get('domain')
case_ids = options.get('case_id')
db = options.get('db')
self.log_filename = 'undo_uuid_clash.{}.log'.format(
datetime.utcnow().isoformat())
print('\nWriting output to log file: {}\n'.format(self.log_filename))
if case_ids:
form_ids = set()
for case in CaseAccessorSQL.get_cases(case_ids):
assert not domain or case.domain == domain, 'Case "%s" not in domain "%s"' % (
case.case_id, domain)
form_ids.update(case.xform_ids)
with self:
check_and_process_forms(form_ids, self)
else:
if domain:
domains = [domain]
else:
domains = iter_domains()
for domain in domains:
print(u"Checking domain: %s" % domain)
form_ids_to_check = set()
dbs = [db] if db else get_db_aliases_for_partitioned_query()
for dbname in dbs:
form_ids_to_check.update(
XFormInstanceSQL.objects.using(dbname).filter(
domain=domain,
state=XFormInstanceSQL.DEPRECATED).values_list(
'orig_id', flat=True))
print(' Found %s forms to check' % len(form_ids_to_check))
with self:
for chunk in chunked(form_ids_to_check, 500):
check_and_process_forms(chunk, self)
def __enter__(self):
self._log_file = open(self.log_filename, 'w')
def __exit__(self, exc_type, exc_val, exc_tb):
self._log_file.close()
def log(message):
self._log_file.write(message)
def get_case_ids_for_reassignment(domain, location_id):
"""
:return: for cases that belong to location_id return
a dict mapping for all case ids under a household id and
a set of all other case ids
"""
all_case_ids = CaseAccessorSQL.get_case_ids_in_domain_by_owners(domain, [location_id])
other_case_ids = set(all_case_ids)
child_case_ids_per_household_id = {}
for household_case_id in get_household_case_ids(domain, location_id):
household_child_case_ids = get_household_child_case_ids_by_owner(
domain, household_case_id, location_id)
other_case_ids.remove(household_case_id)
other_case_ids = other_case_ids - set(household_child_case_ids)
child_case_ids_per_household_id[household_case_id] = household_child_case_ids
return child_case_ids_per_household_id, other_case_ids
def handle(self, domain, *, state_dir, commit, debug, **options):
if not should_use_sql_backend(domain):
raise CommandError(
f'Cannot unsort commits on couch domain: {domain}')
assert Domain.get_by_name(domain), f'Unknown domain "{domain}"'
setup_logging(state_dir, "unsort_sql_cases", debug)
if commit:
log.info("COMMIT MODE: show and save unsorted transactions...")
else:
log.info("DRY RUN: show but do not save unsorted transactions...")
case_ids = iter_sql_cases_with_sorted_transactions(domain)
for batch in chunked(case_ids, 100, list):
for sql_case in CaseAccessorSQL.get_cases(batch):
unsort_transactions(sql_case, commit)
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for ledger in iter_all_rows(LedgerReindexAccessor()):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def process_bulk_docs(self, docs):
updates = {}
for doc in docs:
case_id = doc['_id']
mother_case_ids = [i.referenced_id for i in CaseAccessorSQL.get_indices(self.domain, case_id)
if i.identifier == MOTHER_INDEX_IDENTIFIER]
if len(mother_case_ids) == 1:
try:
mother_case = self.case_accessor.get_case(mother_case_ids[0])
except CaseNotFound:
pass
else:
updates[case_id] = mother_case.name
if updates:
submit_case_blocks(self._create_case_blocks(updates, MOTHER_NAME_PROPERTY),
self.domain, user_id=SYSTEM_USER_ID)
return True
def _diff_cases(self, couch_cases):
from corehq.apps.tzmigration.timezonemigration import json_diff
log.debug('Calculating case diffs for {} cases'.format(
len(couch_cases)))
statedb = self.statedb
counts = defaultdict(int)
case_ids = list(couch_cases)
sql_cases = CaseAccessorSQL.get_cases(case_ids)
sql_case_ids = set()
for sql_case in sql_cases:
sql_case_ids.add(sql_case.case_id)
couch_case = couch_cases[sql_case.case_id]
sql_case_json = sql_case.to_json()
diffs = json_diff(couch_case,
sql_case_json,
track_list_indices=False)
diffs = filter_case_diffs(couch_case, sql_case_json, diffs,
statedb)
if diffs and not sql_case.is_deleted:
try:
couch_case, diffs = self._rebuild_couch_case_and_re_diff(
couch_case, sql_case_json)
except Exception as err:
log.warning('Case {} rebuild -> {}: {}'.format(
sql_case.case_id,
type(err).__name__, err))
if diffs:
statedb.add_diffs(couch_case['doc_type'], sql_case.case_id,
diffs)
counts[couch_case['doc_type']] += 1
self._diff_ledgers(case_ids)
if len(case_ids) != len(sql_case_ids):
couch_ids = set(case_ids)
assert not (sql_case_ids - couch_ids), sql_case_ids - couch_ids
missing_cases = [couch_cases[x] for x in couch_ids - sql_case_ids]
log.debug("Found %s missing SQL cases", len(missing_cases))
for doc_type, doc_ids in self._filter_missing_cases(missing_cases):
statedb.add_missing_docs(doc_type, doc_ids)
counts[doc_type] += len(doc_ids)
for doc_type, count in six.iteritems(counts):
statedb.increment_counter(doc_type, count)
self.processed_docs += len(case_ids)
self._log_case_diff_count(throttled=True)
def get_case_ids_for_reassignment(domain, location_id):
"""
:return: for cases that belong to location_id return
a dict mapping for all case ids under a household id and
a list of all other case ids
"""
all_case_ids = CaseAccessorSQL.get_case_ids_in_domain_by_owners(domain, [location_id])
all_cases = CaseAccessors(domain).get_cases(all_case_ids)
other_case_ids = set([case.case_id for case in all_cases if case.type not in CASE_TYPES_TO_IGNORE])
child_case_ids_per_household_id = {}
for household_case_id in get_household_case_ids(domain, location_id):
household_child_case_ids = get_household_child_case_ids_by_owner(
domain, household_case_id, location_id)
other_case_ids.remove(household_case_id)
other_case_ids = other_case_ids - set(household_child_case_ids)
child_case_ids_per_household_id[household_case_id] = household_child_case_ids
return child_case_ids_per_household_id, list(other_case_ids)
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for db in get_sql_db_aliases_in_use():
for ledger in LedgerReindexAccessor().get_docs(db, None, limit=10000):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def get_case_transactions(case_id, updated_xforms=None):
"""
This fetches all the transactions required to rebuild the case along
with all the forms for those transactions.
For any forms that have been updated it replaces the old form
with the new one.
:param case_id: ID of case to rebuild
:param updated_xforms: list of forms that have been changed.
:return: list of ``CaseTransaction`` objects with their associated forms attached.
"""
transactions = CaseAccessorSQL.get_transactions_for_case_rebuild(case_id)
form_ids = {tx.form_id for tx in transactions}
updated_xforms_map = {
xform.form_id: xform
for xform in updated_xforms if not xform.is_deprecated
} if updated_xforms else {}
form_ids_to_fetch = list(form_ids - set(updated_xforms_map.keys()))
xform_map = {
form.form_id: form
for form in FormAccessorSQL.get_forms_with_attachments_meta(
form_ids_to_fetch)
}
def get_form(form_id):
if form_id in updated_xforms_map:
return updated_xforms_map[form_id]
try:
return xform_map[form_id]
except KeyError:
raise XFormNotFound
for transaction in transactions:
if transaction.form_id:
try:
transaction.cached_form = get_form(transaction.form_id)
except XFormNotFound:
logging.error('Form not found during rebuild: %s',
transaction.form_id)
return transactions
