def test_ledger_pillow(self): factory = CaseFactory(domain=self.domain) case = factory.create_case() consumer = get_test_kafka_consumer(topics.LEDGER) # have to get the seq id before the change is processed kafka_seq = get_topic_offset(topics.LEDGER) from corehq.apps.commtrack.tests.util import get_single_balance_block from corehq.apps.hqcase.utils import submit_case_blocks submit_case_blocks([ get_single_balance_block(case.case_id, self.product_id, 100)], self.domain ) ref = UniqueLedgerReference(case.case_id, 'stock', self.product_id) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) if should_use_sql_backend(self.domain): self.assertEqual(ref.as_id(), change_meta.document_id) else: from corehq.apps.commtrack.models import StockState state = StockState.objects.all() self.assertEqual(1, len(state)) self.assertEqual(state[0].pk, change_meta.document_id) # self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(LEDGER_INDEX_INFO.index) # confirm change made it to elasticserach self._assert_ledger_in_es(ref)
def test_ledger_pillow_sql(self): factory = CaseFactory(domain=self.domain) case = factory.create_case() consumer = get_test_kafka_consumer(topics.LEDGER) # have to get the seq id before the change is processed kafka_seq = consumer.offsets()['fetch'][(topics.LEDGER, 0)] from corehq.apps.commtrack.tests import get_single_balance_block from corehq.apps.hqcase.utils import submit_case_blocks submit_case_blocks([ get_single_balance_block(case.case_id, self.product_id, 100)], self.domain ) ref = UniqueLedgerReference(case.case_id, 'stock', self.product_id) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) if should_use_sql_backend(self.domain): self.assertEqual(ref.as_id(), change_meta.document_id) else: from corehq.apps.commtrack.models import StockState state = StockState.objects.all() self.assertEqual(1, len(state)) self.assertEqual(state[0].pk, change_meta.document_id) # self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(LEDGER_INDEX_INFO.index) # confirm change made it to elasticserach results = self.elasticsearch.search( LEDGER_INDEX_INFO.index, LEDGER_INDEX_INFO.type, body={ "query": { "bool": { "must": [{ "match_all": {} }] } } } ) self.assertEqual(1, results['hits']['total']) ledger_doc = results['hits']['hits'][0]['_source'] self.assertEqual(self.domain, ledger_doc['domain']) self.assertEqual(ref.case_id, ledger_doc['case_id']) self.assertEqual(ref.section_id, ledger_doc['section_id']) self.assertEqual(ref.entry_id, ledger_doc['entry_id'])
def get_doc(self, doc_id): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference ref = UniqueLedgerReference.from_id(doc_id) try: return LedgerAccessorSQL.get_ledger_value(**ref._asdict()) except CaseNotFound: pass
def get_document(self, doc_id): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference try: ref = UniqueLedgerReference.from_id(doc_id) return self.ledger_accessors.get_ledger_value(**ref._asdict()).to_json() except LedgerValueNotFound as e: raise DocumentNotFoundError(e)
def get_document(self, doc_id): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference try: ref = UniqueLedgerReference.from_id(doc_id) return self.ledger_accessors.get_ledger_value( **ref._asdict()).to_json() except LedgerValueNotFound as e: raise DocumentNotFoundError(e)
def _couch_iterator(self): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference case_accessors = CaseAccessors(domain=self.domain) # assuming we're only interested in the 'stock' section for now for case_id in case_accessors.get_case_ids_in_domain(): for product_id in self.product_ids: yield UniqueLedgerReference(case_id, 'stock', product_id).to_id()
def iter_document_ids(self, last_id=None): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference # todo: support last_id # assuming we're only interested in the 'stock' section for now for case_id in self.case_accessors.get_case_ids_in_domain(self.domain): for product_id in self.product_ids: yield UniqueLedgerReference(case_id, 'stock', product_id).to_id()
def get_ledger_references_from_stock_transactions(xform): stock_report_helpers = list(get_all_stock_report_helpers_from_form(xform)) return { UniqueLedgerReference(tx_helper.case_id, tx_helper.section_id, tx_helper.product_id) for stock_report_helper in stock_report_helpers for tx_helper in stock_report_helper.transactions }
def __init__(self, diff, ref=None): if ref is None: ref = UniqueLedgerReference.from_id(diff.doc_id) diff = diff.json_diff else: assert not isinstance(diff.path, str), (ref, diff) self.ref = ref for name in FormJsonDiff._fields: setattr(self, name, getattr(diff, name))
def iter_documents(self, ids): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference case_id_map = defaultdict(list) for id_string in ids: case_id, section_id, entry_id = UniqueLedgerReference.from_id(id_string) case_id_map[(section_id, entry_id)].append(case_id) for section_entry, case_ids in case_id_map.iteritems(): section_id, entry_id = section_entry results = self.ledger_accessors.get_ledger_values_for_cases(case_ids, section_id, entry_id) for ledger_value in results: yield ledger_value.to_json()
def add_ledger(self, case_id, **values): ref = UniqueLedgerReference(case_id, "stock", case_id) self.sql_ledgers[case_id] = Config( ledger_reference=ref, values=values, last_modified_form_id="form", to_json=lambda: dict(values, ledger_reference=ref.as_id()), ) couch_values = dict(values) stock = Config( ledger_reference=ref, values=couch_values, last_modified_form_id="form", to_json=lambda: dict(couch_values, ledger_reference=ref.as_id()), ) self.couch_ledgers[case_id] = stock tx = Config( report=Config(form_id="form", type="transfer"), ledger_reference=ref, ) tx_helper = Config(ledger_reference=ref) self.stock_transactions[ref] = [tx] self.form_transactions["form"] = [("transfer", tx_helper)] return stock
def setUp(self): super(RebuildStockStateTest, self).setUp() self.domain = 'asldkjf-domain' self.case = CaseFactory(domain=self.domain).create_case() self.product = make_product(self.domain, 'Product Name', 'prodcode') self._stock_state_key = dict( section_id='stock', case_id=self.case.case_id, product_id=self.product.get_id ) self.unique_reference = UniqueLedgerReference( case_id=self.case.case_id, section_id='stock', entry_id=self.product.get_id ) self.ledger_processor = FormProcessorInterface(self.domain).ledger_processor
def create_docs(cls, domain, count): from corehq.apps.commtrack.tests.util import get_single_balance_block from corehq.apps.hqcase.utils import submit_case_blocks from corehq.apps.commtrack.helpers import make_product from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference cls.product = make_product(cls.domain, 'A Product', 'prodcode_a') factory = CaseFactory(cls.domain) case_ids = [factory.create_case().case_id for i in range(count)] for case_id in case_ids: submit_case_blocks([ get_single_balance_block(case_id, cls.product._id, 10) ], domain) return [ UniqueLedgerReference(case_id, 'stock', cls.product._id).as_id() for case_id in case_ids ]
def ledger_reference(self): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference return UniqueLedgerReference( case_id=self.case_id, section_id=self.section_id, entry_id=self.product_id )
self.product_b._id: 50, self.product_c._id: 25, } self._submit_ledgers([ get_single_balance_block(self.case.case_id, prod_id, balance) for prod_id, balance in balances.items() ]) expected_transactions = [] for prod_id, expected_balance in balances.items(): expected_transactions.append(self._expected_val( expected_balance, expected_balance, product_id=prod_id )) balance = self.interface.ledger_db.get_current_ledger_value( UniqueLedgerReference( case_id=self.case.case_id, section_id='stock', entry_id=prod_id ) ) self.assertEqual(expected_balance, balance) self._assert_transactions(expected_transactions, ignore_ordering=True) def test_balance_submission_with_prior_balance(self): self._set_balance(100) self._assert_ledger_state(100) self._set_balance(50) self._assert_ledger_state(50) self._set_balance(150) self._assert_ledger_state(150)
class RebuildStockStateTest(TestCase): def setUp(self): super(RebuildStockStateTest, self).setUp() self.domain = 'asldkjf-domain' self.case = CaseFactory(domain=self.domain).create_case() self.product = make_product(self.domain, 'Product Name', 'prodcode') self._stock_state_key = dict( section_id='stock', case_id=self.case.case_id, product_id=self.product.get_id ) self.unique_reference = UniqueLedgerReference( case_id=self.case.case_id, section_id='stock', entry_id=self.product.get_id ) self.ledger_processor = FormProcessorInterface(self.domain).ledger_processor def _assert_stats(self, epxected_tx_count, expected_stock_state_balance, expected_tx_balance): ledger_value = LedgerAccessors(self.domain).get_ledger_value(**self.unique_reference._asdict()) latest_txn = LedgerAccessors(self.domain).get_latest_transaction(**self.unique_reference._asdict()) all_txns = LedgerAccessors(self.domain).get_ledger_transactions_for_case(**self.unique_reference._asdict()) self.assertEqual(epxected_tx_count, len(all_txns)) self.assertEqual(expected_stock_state_balance, ledger_value.stock_on_hand) self.assertEqual(expected_tx_balance, latest_txn.stock_on_hand) def _submit_ledgers(self, ledger_blocks): return submit_case_blocks( ledger_blocks.format(**self._stock_state_key), self.domain)[0].form_id @run_with_all_backends def test_simple(self): self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) self._assert_stats(2, 100, 100) self.ledger_processor.rebuild_ledger_state(**self.unique_reference._asdict()) self._assert_stats(2, 200, 200) def test_inferred(self): self._submit_ledgers(LEDGER_BLOCKS_INFERRED) # this is weird behavior: # it just doesn't process the second one # even though knowing yesterday's certainly changes the meaning # of today's transfer # UPDATE SK 2016-03-14: this happens because the transactions are received out of order # (they appear out of order in the form XML) and hence saved out of order. # When the older transaction is saved it will only look back in time # to create inferred transactions and not ahead. self._assert_stats(2, 50, 50) rebuild_stock_state(**self._stock_state_key) self._assert_stats(2, 150, 150) @run_with_all_backends def test_case_actions(self): # make sure that when a case is rebuilt (using rebuild_case) # stock transactions show up as well form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) case_id = self.case.case_id rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test')) case = CaseAccessors(self.domain).get_case(self.case.case_id) self.assertEqual(case.xform_ids[1:], [form_id]) self.assertTrue(form_id in [action.form_id for action in case.actions]) @run_with_all_backends def test_edit_submissions_simple(self): initial_quantity = 100 form = submit_case_blocks( case_blocks=get_single_balance_block(quantity=initial_quantity, **self._stock_state_key), domain=self.domain, )[0] self._assert_stats(1, initial_quantity, initial_quantity) case_accessors = CaseAccessors(self.domain) case = case_accessors.get_case(self.case.case_id) try: self.assertTrue(any([action.is_ledger_transaction for action in case.actions])) except AttributeError: self.assertTrue('commtrack' in [action.action_type for action in case.actions]) self.assertEqual([form.form_id], case.xform_ids[1:]) # change the value to 50 edit_quantity = 50 submit_case_blocks( case_blocks=get_single_balance_block(quantity=edit_quantity, **self._stock_state_key), domain=self.domain, form_id=form.form_id, ) case = case_accessors.get_case(self.case.case_id) try: # CaseTransaction self.assertTrue(any([action.is_ledger_transaction for action in case.actions])) except AttributeError: # CaseAction self.assertTrue('commtrack' in [action.action_type for action in case.actions]) self._assert_stats(1, edit_quantity, edit_quantity) self.assertEqual([form.form_id], case.xform_ids[1:])
class RebuildStockStateTest(TestCase): def setUp(self): super(RebuildStockStateTest, self).setUp() self.domain = 'asldkjf-domain' self.case = CaseFactory(domain=self.domain).create_case() self.product = make_product(self.domain, 'Product Name', 'prodcode') self._stock_state_key = dict(section_id='stock', case_id=self.case.case_id, product_id=self.product.get_id) self.unique_reference = UniqueLedgerReference( case_id=self.case.case_id, section_id='stock', entry_id=self.product.get_id) self.ledger_processor = FormProcessorInterface( self.domain).ledger_processor def _assert_stats(self, epxected_tx_count, expected_stock_state_balance, expected_tx_balance): ledger_value = LedgerAccessors( self.domain).get_ledger_value(**self.unique_reference._asdict()) latest_txn = LedgerAccessors(self.domain).get_latest_transaction( **self.unique_reference._asdict()) all_txns = LedgerAccessors( self.domain).get_ledger_transactions_for_case( **self.unique_reference._asdict()) self.assertEqual(epxected_tx_count, len(all_txns)) self.assertEqual(expected_stock_state_balance, ledger_value.stock_on_hand) self.assertEqual(expected_tx_balance, latest_txn.stock_on_hand) def _submit_ledgers(self, ledger_blocks): return submit_case_blocks( ledger_blocks.format(**self._stock_state_key), self.domain)[0].form_id @run_with_all_backends def test_simple(self): self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) self._assert_stats(2, 100, 100) self.ledger_processor.rebuild_ledger_state( **self.unique_reference._asdict()) self._assert_stats(2, 200, 200) def test_inferred(self): self._submit_ledgers(LEDGER_BLOCKS_INFERRED) # this is weird behavior: # it just doesn't process the second one # even though knowing yesterday's certainly changes the meaning # of today's transfer # UPDATE SK 2016-03-14: this happens because the transactions are received out of order # (they appear out of order in the form XML) and hence saved out of order. # When the older transaction is saved it will only look back in time # to create inferred transactions and not ahead. self._assert_stats(2, 50, 50) rebuild_stock_state(**self._stock_state_key) self._assert_stats(2, 150, 150) @run_with_all_backends def test_case_actions(self): # make sure that when a case is rebuilt (using rebuild_case) # stock transactions show up as well form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) case_id = self.case.case_id rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test')) case = CaseAccessors(self.domain).get_case(self.case.case_id) self.assertEqual(case.xform_ids[1:], [form_id]) self.assertTrue(form_id in [action.form_id for action in case.actions]) @run_with_all_backends def test_edit_submissions_simple(self): initial_quantity = 100 form = submit_case_blocks( case_blocks=get_single_balance_block(quantity=initial_quantity, **self._stock_state_key), domain=self.domain, )[0] self._assert_stats(1, initial_quantity, initial_quantity) case_accessors = CaseAccessors(self.domain) case = case_accessors.get_case(self.case.case_id) try: self.assertTrue( any([action.is_ledger_transaction for action in case.actions])) except AttributeError: self.assertTrue( 'commtrack' in [action.action_type for action in case.actions]) self.assertEqual([form.form_id], case.xform_ids[1:]) # change the value to 50 edit_quantity = 50 submit_case_blocks( case_blocks=get_single_balance_block(quantity=edit_quantity, **self._stock_state_key), domain=self.domain, form_id=form.form_id, ) case = case_accessors.get_case(self.case.case_id) try: # CaseTransaction self.assertTrue( any([action.is_ledger_transaction for action in case.actions])) except AttributeError: # CaseAction self.assertTrue( 'commtrack' in [action.action_type for action in case.actions]) self._assert_stats(1, edit_quantity, edit_quantity) self.assertEqual([form.form_id], case.xform_ids[1:])
class RebuildStockStateTest(TestCase): def setUp(self): super(RebuildStockStateTest, self).setUp() self.domain = 'asldkjf-domain' self.case = CaseFactory(domain=self.domain).create_case() self.product = make_product(self.domain, 'Product Name', 'prodcode') self._stock_state_key = dict(section_id='stock', case_id=self.case.case_id, product_id=self.product.get_id) self.unique_reference = UniqueLedgerReference( case_id=self.case.case_id, section_id='stock', entry_id=self.product.get_id) self.ledger_processor = FormProcessorInterface( self.domain).ledger_processor def _assert_stats(self, epxected_tx_count, expected_stock_state_balance, expected_tx_balance): ledger_value = LedgerAccessors( self.domain).get_ledger_value(**self.unique_reference._asdict()) latest_txn = LedgerAccessors(self.domain).get_latest_transaction( **self.unique_reference._asdict()) all_txns = LedgerAccessors( self.domain).get_ledger_transactions_for_case( **self.unique_reference._asdict()) self.assertEqual(epxected_tx_count, len(all_txns)) self.assertEqual(expected_stock_state_balance, ledger_value.stock_on_hand) self.assertEqual(expected_tx_balance, latest_txn.stock_on_hand) def _submit_ledgers(self, ledger_blocks): return submit_case_blocks( ledger_blocks.format(**self._stock_state_key), self.domain)[0].form_id def test_simple(self): self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) self._assert_stats(2, 100, 100) self.ledger_processor.rebuild_ledger_state( **self.unique_reference._asdict()) self._assert_stats(2, 200, 200) def test_case_actions(self): # make sure that when a case is rebuilt (using rebuild_case) # stock transactions show up as well form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) case_id = self.case.case_id rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test')) case = CommCareCase.objects.get_case(self.case.case_id, self.domain) self.assertEqual(case.xform_ids[1:], [form_id]) self.assertTrue(form_id in [action.form_id for action in case.actions]) @softer_assert() def test_edit_submissions_simple(self): initial_quantity = 100 form = submit_case_blocks( case_blocks=get_single_balance_block(quantity=initial_quantity, **self._stock_state_key), domain=self.domain, )[0] self._assert_stats(1, initial_quantity, initial_quantity) case = CommCareCase.objects.get_case(self.case.case_id, self.domain) try: self.assertTrue( any([action.is_ledger_transaction for action in case.actions])) except AttributeError: self.assertTrue( 'commtrack' in [action.action_type for action in case.actions]) self.assertEqual([form.form_id], case.xform_ids[1:]) # change the value to 50 edit_quantity = 50 submit_case_blocks( case_blocks=get_single_balance_block(quantity=edit_quantity, **self._stock_state_key), domain=self.domain, form_id=form.form_id, ) case = CommCareCase.objects.get_case(self.case.case_id, self.domain) try: # CaseTransaction self.assertTrue( any([action.is_ledger_transaction for action in case.actions])) except AttributeError: # CaseAction self.assertTrue( 'commtrack' in [action.action_type for action in case.actions]) self._assert_stats(1, edit_quantity, edit_quantity) self.assertEqual([form.form_id], case.xform_ids[1:])