def test_reprocess_unfinished_submission_ledger_create(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id,
attrs={
'case_type': 'parent',
'create': True
})
])
transaction_patch = patch(
'corehq.form_processor.backends.sql.processor.transaction')
ledger_save_patch = patch(
'corehq.form_processor.backends.sql.dbaccessors.LedgerAccessorSQL.save_ledger_values',
side_effect=InternalError)
with transaction_patch, ledger_save_patch, self.assertRaises(
InternalError):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 100),
self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id)
self.assertEqual(0, len(ledgers))
# case transaction got saved
case = CaseAccessorSQL.get_case(case_id)
self.assertEqual(2, len(case.transactions))
self.assertTrue(case.transactions[0].is_case_create)
self.assertTrue(case.transactions[1].is_ledger_transaction)
ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
case_id)
self.assertEqual(0, len(ledger_transactions))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(0, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
case_id)
self.assertEqual(1, len(ledger_transactions))
# case still only has 2 transactions
case = CaseAccessorSQL.get_case(case_id)
self.assertEqual(2, len(case.transactions))
def test_migrate_ledgers(self):
case_id = uuid.uuid4().hex
create_and_save_a_case(self.domain_name,
case_id=case_id,
case_name="Simon's sweet shop")
self._set_balance(100,
case_id,
self.liquorice._id,
type="set_the_liquorice_balance")
self._set_balance(50, case_id, self.sherbert._id)
self._set_balance(175, case_id, self.jelly_babies._id)
expected_stock_state = {
'stock': {
self.liquorice._id: 100,
self.sherbert._id: 50,
self.jelly_babies._id: 175
}
}
self._validate_ledger_data(self._get_ledger_state(case_id),
expected_stock_state)
self._do_migration_and_assert_flags(self.domain_name)
self._validate_ledger_data(self._get_ledger_state(case_id),
expected_stock_state)
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
case_id)
self.assertEqual(3, len(transactions))
self._compare_diffs([])
def testStockReportRoaming(self):
self.assertEqual([], list(iter_commtrack_forms(self.domain.name)))
amounts = {
'pp': 10,
'pq': 20,
'pr': 30,
}
# soh loc1 pp 10 pq 20...
handled = handle(
get_two_way_number_for_recipient(self.users[0]),
'soh {loc} {report}'.format(
loc='loc1',
report=' '.join('%s %s' % (k, v) for k, v in amounts.items())),
None)
self.assertTrue(handled)
forms = list(iter_commtrack_forms(self.domain.name))
self.assertEqual(1, len(forms))
self.assertEqual(_get_location_from_sp(self.sp),
_get_location_from_form(forms[0]))
ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
self.sp.case_id)
self.assertEqual({forms[0].form_id},
set(t.form_id for t in ledger_transactions))
self.assertEqual({'balance'},
set(t.readable_type for t in ledger_transactions))
self.assertEqual(3, len(ledger_transactions))
self.check_transaction_amounts(ledger_transactions, amounts)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form(
[case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
def hard_rebuild_ledgers(case_id, section_id=None, entry_id=None):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id, section_id, entry_id)
if not transactions:
LedgerAccessorSQL.delete_ledger_values(case_id, section_id, entry_id)
return
ledger_value = LedgerAccessorSQL.get_ledger_value(case_id, section_id, entry_id)
ledger_value = LedgerProcessorSQL._rebuild_ledger_value_from_transactions(ledger_value, transactions)
LedgerAccessorSQL.save_ledger_values([ledger_value])
publish_ledger_v2_saved(ledger_value)
def hard_rebuild_ledgers(domain, case_id, section_id, entry_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id, section_id, entry_id)
if not transactions:
LedgerAccessorSQL.delete_ledger_values(case_id, section_id, entry_id)
publish_ledger_v2_deleted(domain, case_id, section_id, entry_id)
return
ledger_value = LedgerAccessorSQL.get_ledger_value(case_id, section_id, entry_id)
ledger_value = LedgerProcessorSQL._rebuild_ledger_value_from_transactions(ledger_value, transactions)
LedgerAccessorSQL.save_ledger_values([ledger_value])
publish_ledger_v2_saved(ledger_value)
def check_form_type(self, is_consumption):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
self.sp.case_id)
transactions = [
t for t in transactions if t.readable_type != 'balance'
]
self.assertEqual(3, len(transactions))
for transaction in transactions:
if is_consumption:
self.assertLess(transaction.delta, 0)
else:
self.assertGreater(transaction.delta, 0)
def test_delete_ledger_transactions_for_form(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
self._set_balance(100, self.case_one.case_id, self.product_a._id)
case_ids = [self.case_one.case_id, self.case_two.case_id]
form_id = self._submit_ledgers([
get_single_balance_block(case_id, product_id, 10)
for case_id in case_ids
for product_id in [self.product_a._id, self.product_b._id]
])
deleted = LedgerAccessorSQL.delete_ledger_transactions_for_form(case_ids, form_id)
self.assertEqual(4, deleted)
self.assertEqual(
1,
len(LedgerAccessorSQL.get_ledger_transactions_for_case(self.case_one.case_id))
)
self.assertEqual(
0,
len(LedgerAccessorSQL.get_ledger_transactions_for_case(self.case_two.case_id))
)
def _assert_transactions(self, values, ignore_ordering=False):
if should_use_sql_backend(DOMAIN):
txs = LedgerAccessorSQL.get_ledger_transactions_for_case(self.case.case_id)
self.assertEqual(len(values), len(txs))
if ignore_ordering:
values = sorted(values, key=lambda v: (v.type, v.product_id))
txs = sorted(txs, key=lambda t: (t.type, t.entry_id))
for expected, tx in zip(values, txs):
self.assertEqual(expected.type, tx.type)
self.assertEqual(expected.product_id, tx.entry_id)
self.assertEqual('stock', tx.section_id)
self.assertEqual(expected.delta, tx.delta)
self.assertEqual(expected.updated_balance, tx.updated_balance)
def delete_all_ledgers(domain):
if should_use_sql_backend(domain):
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
else:
from casexml.apps.stock.models import StockReport
from casexml.apps.stock.models import StockTransaction
stock_report_ids = StockReport.objects.filter(domain=domain).values_list('id', flat=True)
StockReport.objects.filter(domain=domain).delete()
StockTransaction.objects.filter(report_id__in=stock_report_ids).delete()
def test_ledgers(self):
expected_object_counts = Counter({
XFormInstanceSQL: 3,
BlobMeta: 3,
CommCareCaseSQL: 1,
CaseTransaction: 3,
LedgerValue: 1,
LedgerTransaction: 2
})
case = self.factory.create_case()
submit_case_blocks([
get_single_balance_block(case.case_id, self.product._id, 10)
], self.domain_name)
submit_case_blocks([
get_single_balance_block(case.case_id, self.product._id, 5)
], self.domain_name)
pre_ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(case.case_id)
pre_ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case.case_id)
self.assertEqual(1, len(pre_ledger_values))
self.assertEqual(2, len(pre_ledger_transactions))
self._dump_and_load(expected_object_counts)
post_ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(case.case_id)
post_ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case.case_id)
self.assertEqual(1, len(post_ledger_values))
self.assertEqual(2, len(post_ledger_transactions))
self.assertEqual(pre_ledger_values[0].ledger_reference, post_ledger_values[0].ledger_reference)
self.assertDictEqual(pre_ledger_values[0].to_json(), post_ledger_values[0].to_json())
pre_ledger_transactions = sorted(pre_ledger_transactions, key=lambda t: t.pk)
post_ledger_transactions = sorted(post_ledger_transactions, key=lambda t: t.pk)
for pre, post in zip(pre_ledger_transactions, post_ledger_transactions):
self.assertEqual(str(pre), str(post))
def test_delete_ledger_transactions_for_form(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
self._set_balance(100, self.case_one.case_id, self.product_a._id)
case_ids = [self.case_one.case_id, self.case_two.case_id]
form_id = self._submit_ledgers([
get_single_balance_block(case_id, product_id, 10)
for case_id in case_ids
for product_id in [self.product_a._id, self.product_b._id]
])
deleted = LedgerAccessorSQL.delete_ledger_transactions_for_form(
case_ids, form_id)
self.assertEqual(4, deleted)
self.assertEqual(
1,
len(
LedgerAccessorSQL.get_ledger_transactions_for_case(
self.case_one.case_id)))
self.assertEqual(
0,
len(
LedgerAccessorSQL.get_ledger_transactions_for_case(
self.case_two.case_id)))
def _rebuild_ledger(self, form_id, ledger_value):
"""
Rebuild a LedgerValue and its associated transactions during a form edit workflow.
:param form_id: ID of edited form
:param ledger_value: LedgerValue to rebuild with transactions from new form tracked on the model
:return: updated LedgerValue object
"""
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(**ledger_value.ledger_reference._asdict())
transaction_excluding_deprecated_form = [tx for tx in transactions if tx.form_id != form_id]
new_transactions = ledger_value.get_tracked_models_to_create(LedgerTransaction)
all_transactions = transaction_excluding_deprecated_form + new_transactions
sorted_transactions = sorted(all_transactions, key=lambda t: t.report_date)
ledger_value.clear_tracked_models(LedgerTransaction)
ledger_value = self._rebuild_ledger_value_from_transactions(ledger_value, sorted_transactions)
return ledger_value
def _rebuild_ledger(self, form_id, ledger_value):
"""
Rebuild a LedgerValue and its associated transactions during a form edit workflow.
:param form_id: ID of edited form
:param ledger_value: LedgerValue to rebuild with transactions from new form tracked on the model
:return: updated LedgerValue object
"""
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
**ledger_value.ledger_reference._asdict()
)
transaction_excluding_deprecated_form = [tx for tx in transactions if tx.form_id != form_id]
new_transactions = ledger_value.get_tracked_models_to_create(LedgerTransaction)
all_transactions = transaction_excluding_deprecated_form + new_transactions
sorted_transactions = sorted(all_transactions, key=lambda t: t.report_date)
ledger_value.clear_tracked_models(LedgerTransaction)
ledger_value = self._rebuild_ledger_value_from_transactions(ledger_value, sorted_transactions)
return ledger_value
def test_migrate_ledgers(self):
case_id = uuid.uuid4().hex
create_and_save_a_case(self.domain_name, case_id=case_id, case_name="Simon's sweet shop")
self._set_balance(100, case_id, self.liquorice._id)
self._set_balance(50, case_id, self.sherbert._id)
self._set_balance(175, case_id, self.jelly_babies._id)
expected_stock_state = {'stock': {
self.liquorice._id: 100,
self.sherbert._id: 50,
self.jelly_babies._id: 175
}}
self._validate_ledger_data(self._get_ledger_state(case_id), expected_stock_state)
self._do_migration_and_assert_flags(self.domain_name)
self._validate_ledger_data(self._get_ledger_state(case_id), expected_stock_state)
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
self.assertEqual(3, len(transactions))
self._compare_diffs([])
def testStockReportFixed(self):
self.assertEqual([], list(iter_commtrack_forms(self.domain.name)))
amounts = {
'pp': 10,
'pq': 20,
'pr': 30,
}
# soh loc1 pp 10 pq 20...
handled = handle(
get_two_way_number_for_recipient(self.users[1]),
'soh {report}'.format(report=' '.join(
'%s %s' % (k, v) for k, v in amounts.items())), None)
self.assertTrue(handled)
forms = list(iter_commtrack_forms(self.domain.name))
self.assertEqual(1, len(forms))
self.assertEqual(_get_location_from_sp(self.sp),
_get_location_from_form(forms[0]))
ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
self.sp.case_id)
self.check_transaction_amounts(ledger_transactions, amounts)
def get_sql_transactions(case_id, section_id, entry_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
case_id=case_id, section_id=section_id, entry_id=entry_id)
return sorted(transactions, key=lambda t: (t.report_date, t.id))
def test_reprocess_unfinished_submission_ledger_rebuild(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(
submit_case_blocks([
CaseBlock(case_id=case_id, create=True,
case_type='shop').as_string(),
get_single_balance_block(case_id, 'product1', 100),
], self.domain)[0].form_id)
transaction_patch = patch(
'corehq.form_processor.backends.sql.processor.transaction')
ledger_save_patch = patch(
'corehq.form_processor.backends.sql.dbaccessors.LedgerAccessorSQL.save_ledger_values',
side_effect=InternalError)
with transaction_patch, ledger_save_patch, self.assertRaises(
InternalError):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 50), self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit another form afterwards
form_ids.append(
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 25),
self.domain)[0].form_id)
ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
case_id)
self.assertEqual(2, len(ledger_transactions))
# should rebuild ledger transactions
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(0, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers)) # still only 1
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
case_id)
self.assertEqual(3, len(ledger_transactions))
# make sure transactions are in correct order
self.assertEqual(form_ids,
[trans.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].updated_balance)
self.assertEqual(100, ledger_transactions[0].delta)
self.assertEqual(50, ledger_transactions[1].updated_balance)
self.assertEqual(-50, ledger_transactions[1].delta)
self.assertEqual(25, ledger_transactions[2].updated_balance)
self.assertEqual(-25, ledger_transactions[2].delta)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
