def process_form_archived(self, form): from corehq.form_processor.parsers.ledgers.form import get_ledger_references_from_stock_transactions refs_to_rebuild = get_ledger_references_from_stock_transactions(form) case_ids = list({ref.case_id for ref in refs_to_rebuild}) LedgerAccessorSQL.delete_ledger_transactions_for_form(case_ids, form.form_id) for ref in refs_to_rebuild: self.rebuild_ledger_state(**ref._asdict())
def _create_ledger(domain, entry_id, balance, case_id=None, section_id='stock'): user_id = 'user1' utcnow = datetime.utcnow() case_id = case_id or uuid.uuid4().hex case = CommCareCaseSQL( case_id=case_id, domain=domain, type='', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, ) CaseAccessorSQL.save_case(case) ledger = LedgerValue( domain=domain, case_id=case_id, section_id=section_id, entry_id=entry_id, balance=balance, last_modified=utcnow ) LedgerAccessorSQL.save_ledger_values([ledger]) return ledger
def _create_models_for_stock_report_helper(self, form, stock_report_helper): processing_result = StockProcessingResult(form, stock_report_helpers=[stock_report_helper]) processing_result.populate_models() if should_use_sql_backend(self.domain_name): from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL LedgerAccessorSQL.save_ledger_values(processing_result.models_to_save) else: processing_result.commit()
def delete_all_ledgers(domain): if should_use_sql_backend(domain): for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id) form_ids = {tx.form_id for tx in transactions} for form_id in form_ids: LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id) LedgerAccessorSQL.delete_ledger_values(case_id) else: from casexml.apps.stock.models import StockReport from casexml.apps.stock.models import StockTransaction stock_report_ids = StockReport.objects.filter(domain=domain).values_list('id', flat=True) StockReport.objects.filter(domain=domain).delete() StockTransaction.objects.filter(report_id__in=stock_report_ids).delete()
def process_form_unarchived(self, form): from corehq.apps.commtrack.processing import process_stock result = process_stock([form]) result.populate_models() LedgerAccessorSQL.save_ledger_values(result.models_to_save) for ledger_value in result.models_to_save: publish_ledger_v2_saved(ledger_value) refs_to_rebuild = {ledger_value.ledger_reference for ledger_value in result.models_to_save} for ref in refs_to_rebuild: self.rebuild_ledger_state(**ref._asdict()) result.finalize()
def update_case_transactions_for_form(case_cache, live_case_updates, deprecated_case_updates, live_form, deprecated_form): for case_update in live_case_updates + deprecated_case_updates: case_id = case_update.id count, _ = CaseTransaction.objects.partitioned_query(case_id)\ .filter(case_id=case_id, form_id=live_form.form_id).delete() rebuild_transactions = CaseTransaction.objects.partitioned_query(case_id).filter( case_id=case_id, type=CaseTransaction.TYPE_REBUILD_FORM_EDIT ) for transaction in rebuild_transactions: if transaction.details.get('deprecated_form_id') == deprecated_form.original_form_id: transaction.delete() for case_update in live_case_updates: case_id = case_update.id case = case_cache.get(case_id) SqlCaseUpdateStrategy.add_transaction_for_form(case, case_update, live_form) for case_update in deprecated_case_updates: case_id = case_update.id case = case_cache.get(case_id) SqlCaseUpdateStrategy.add_transaction_for_form(case, case_update, deprecated_form) stock_result = process_stock([live_form, deprecated_form], case_cache) stock_result.populate_models() affected_ledgers = set() affected_cases = set() ledger_transactions = [] for ledger_value in stock_result.models_to_save: affected_ledgers.add(ledger_value.ledger_reference) affected_cases.add(ledger_value.case_id) for transaction in ledger_value.get_tracked_models_to_create(LedgerTransaction): ledger_transactions.append(transaction) if affected_cases: LedgerAccessorSQL.delete_ledger_transactions_for_form(list(affected_cases), live_form.form_id) for transaction in ledger_transactions: transaction.save() for case in case_cache.cache.values(): affected_cases.add(case.case_id) transactions = case.get_tracked_models_to_create(CaseTransaction) for transaction in transactions: transaction.case = case transaction.save() return affected_cases, affected_ledgers
def test_delete_ledger_values_case_section(self): from corehq.apps.commtrack.tests.util import get_single_balance_block form_id = self._submit_ledgers([ get_single_balance_block(self.case_one.case_id, product_id, 10) for product_id in [self.product_a._id, self.product_b._id] ]) ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id) self.assertEqual(2, len(ledger_values)) LedgerAccessorSQL.delete_ledger_transactions_for_form([self.case_one.case_id], form_id) deleted = LedgerAccessorSQL.delete_ledger_values(self.case_one.case_id, 'stock') self.assertEqual(2, deleted) ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id) self.assertEqual(0, len(ledger_values))
def _get_daily_consumption_for_ledger(ledger): from corehq.apps.commtrack.consumption import get_consumption_for_ledger_json daily_consumption = get_consumption_for_ledger_json(ledger) if should_use_sql_backend(ledger['domain']): from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL ledger_value = LedgerAccessorSQL.get_ledger_value( ledger['case_id'], ledger['section_id'], ledger['entry_id'] ) ledger_value.daily_consumption = daily_consumption LedgerAccessorSQL.save_ledger_values([ledger_value]) else: from corehq.apps.commtrack.models import StockState StockState.objects.filter(pk=ledger['_id']).update(daily_consumption=daily_consumption) return daily_consumption
def get_data(self): from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL locations = self.locations() # locations at this point will only have location objects # that have supply points associated for loc in locations[:self.config.get('max_rows', 100)]: # TODO: this is very inefficient since it loads ALL the transactions up to the supplied # date but only requires the most recent one. Should rather use a window function. transactions = LedgerAccessorSQL.get_ledger_transactions_in_window( case_id=loc.supply_point_id, section_id=SECTION_TYPE_STOCK, entry_id=None, window_start=datetime.min, window_end=self.datetime, ) if self.program_id: transactions = ( tx for tx in transactions if tx.entry_id in self.product_ids ) stock_results = sorted(transactions, key=lambda tx: tx.report_date, reverse=False) yield (loc.name, {tx.entry_id: tx.updated_balance for tx in stock_results})
def get_models_to_update(self, stock_report_helper, ledger_db=None): latest_values = {} for stock_trans in stock_report_helper.transactions: def _lazy_original_balance(): # needs to be in closures because it's zero-argument. # see compute_ledger_values for more information if stock_trans.ledger_reference in latest_values: return latest_values[stock_trans.ledger_reference] else: return self.get_current_ledger_value(stock_trans.ledger_reference) new_ledger_values = compute_ledger_values( _lazy_original_balance, stock_report_helper.report_type, stock_trans.relative_quantity ) latest_values[stock_trans.ledger_reference] = new_ledger_values.balance to_save = [] for touched_ledger_reference, quantity in latest_values.items(): try: ledger_value = LedgerAccessorSQL.get_ledger_value(**touched_ledger_reference._asdict()) except LedgerValue.DoesNotExist: ledger_value = LedgerValue(**touched_ledger_reference._asdict()) ledger_value.balance = quantity to_save.append(ledger_value) return StockModelUpdateResult(to_save=to_save)
def test_delete_ledger_values_case_section_1(self): from corehq.apps.commtrack.tests import get_single_balance_block form_id = self._submit_ledgers([ get_single_balance_block(self.case_one.case_id, self.product_a._id, 10, section_id=section_id) for section_id in ['stock', 'consumption'] ]) ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id) self.assertEqual(2, len(ledger_values)) LedgerAccessorSQL.delete_ledger_transactions_for_form([self.case_one.case_id], form_id) deleted = LedgerAccessorSQL.delete_ledger_values(self.case_one.case_id, 'stock') self.assertEqual(1, deleted) ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id) self.assertEqual(1, len(ledger_values)) self.assertEqual('consumption', ledger_values[0].section_id)
def save_processed_models(cls, processed_forms, cases=None, stock_result=None): with transaction.atomic(): logging.debug('Beginning atomic commit\n') # Save deprecated form first to avoid ID conflicts if processed_forms.deprecated: FormAccessorSQL.save_deprecated_form(processed_forms.deprecated) FormAccessorSQL.save_new_form(processed_forms.submitted) if cases: for case in cases: CaseAccessorSQL.save_case(case) if stock_result: ledgers_to_save = stock_result.models_to_save LedgerAccessorSQL.save_ledger_values(ledgers_to_save, processed_forms.deprecated) cls._publish_changes(processed_forms, cases, stock_result)
def save_processed_models(cls, processed_forms, cases=None, stock_result=None, publish_to_kafka=True): db_names = {processed_forms.submitted.db} if processed_forms.deprecated: db_names |= {processed_forms.deprecated.db} if cases: db_names |= {case.db for case in cases} if stock_result: db_names |= { ledger_value.db for ledger_value in stock_result.models_to_save } with ExitStack() as stack: for db_name in db_names: stack.enter_context(transaction.atomic(db_name)) # Save deprecated form first to avoid ID conflicts if processed_forms.deprecated: FormAccessorSQL.update_form(processed_forms.deprecated, publish_changes=False) FormAccessorSQL.save_new_form(processed_forms.submitted) if cases: for case in cases: CaseAccessorSQL.save_case(case) if stock_result: ledgers_to_save = stock_result.models_to_save LedgerAccessorSQL.save_ledger_values(ledgers_to_save, stock_result) if cases: sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled( processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN) if sort_submissions: for case in cases: if SqlCaseUpdateStrategy(case).reconcile_transactions_if_necessary(): CaseAccessorSQL.save_case(case) if publish_to_kafka: try: cls.publish_changes_to_kafka(processed_forms, cases, stock_result) except Exception as e: raise KafkaPublishingError(e)
def setUp(self): # can't do this in setUpClass until Django 1.9 since @override_settings # doesn't apply to classmethods from corehq.apps.commtrack.tests import get_single_balance_block factory = CaseFactory(domain=self.domain) self.case = factory.create_case() submit_case_blocks([ get_single_balance_block(self.case.case_id, self.product._id, 10) ], self.domain) ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case.case_id) self.assertEqual(1, len(ledger_values))
def test_delete_ledger_transactions_for_form(self): from corehq.apps.commtrack.tests.util import get_single_balance_block self._set_balance(100, self.case_one.case_id, self.product_a._id) case_ids = [self.case_one.case_id, self.case_two.case_id] form_id = self._submit_ledgers([ get_single_balance_block(case_id, product_id, 10) for case_id in case_ids for product_id in [self.product_a._id, self.product_b._id] ]) deleted = LedgerAccessorSQL.delete_ledger_transactions_for_form(case_ids, form_id) self.assertEqual(4, deleted) self.assertEqual( 1, len(LedgerAccessorSQL.get_ledger_transactions_for_case(self.case_one.case_id)) ) self.assertEqual( 0, len(LedgerAccessorSQL.get_ledger_transactions_for_case(self.case_two.case_id)) )
def _assert_transactions(self, values, ignore_ordering=False): if should_use_sql_backend(DOMAIN): txs = LedgerAccessorSQL.get_ledger_transactions_for_case(self.case.case_id) self.assertEqual(len(values), len(txs)) if ignore_ordering: values = sorted(values, key=lambda v: (v.type, v.product_id)) txs = sorted(txs, key=lambda t: (t.type, t.entry_id)) for expected, tx in zip(values, txs): self.assertEqual(expected.type, tx.type) self.assertEqual(expected.product_id, tx.entry_id) self.assertEqual('stock', tx.section_id) self.assertEqual(expected.delta, tx.delta) self.assertEqual(expected.updated_balance, tx.updated_balance)
def test_ledgers(self): expected_object_counts = Counter({ XFormInstanceSQL: 3, BlobMeta: 3, CommCareCaseSQL: 1, CaseTransaction: 3, LedgerValue: 1, LedgerTransaction: 2 }) case = self.factory.create_case() submit_case_blocks([ get_single_balance_block(case.case_id, self.product._id, 10) ], self.domain_name) submit_case_blocks([ get_single_balance_block(case.case_id, self.product._id, 5) ], self.domain_name) pre_ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(case.case_id) pre_ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case.case_id) self.assertEqual(1, len(pre_ledger_values)) self.assertEqual(2, len(pre_ledger_transactions)) self._dump_and_load(expected_object_counts) post_ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(case.case_id) post_ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case.case_id) self.assertEqual(1, len(post_ledger_values)) self.assertEqual(2, len(post_ledger_transactions)) self.assertEqual(pre_ledger_values[0].ledger_reference, post_ledger_values[0].ledger_reference) self.assertDictEqual(pre_ledger_values[0].to_json(), post_ledger_values[0].to_json()) pre_ledger_transactions = sorted(pre_ledger_transactions, key=lambda t: t.pk) post_ledger_transactions = sorted(post_ledger_transactions, key=lambda t: t.pk) for pre, post in zip(pre_ledger_transactions, post_ledger_transactions): self.assertEqual(str(pre), str(post))
def hard_rebuild_ledgers(case_id, section_id=None, entry_id=None): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id, section_id, entry_id) if not transactions: LedgerAccessorSQL.delete_ledger_values(case_id, section_id, entry_id) return ledger_value = LedgerAccessorSQL.get_ledger_value(case_id, section_id, entry_id) ledger_value = LedgerProcessorSQL._rebuild_ledger_value_from_transactions(ledger_value, transactions) LedgerAccessorSQL.save_ledger_values([ledger_value]) publish_ledger_v2_saved(ledger_value)
def _rebuild_ledger(self, form_id, ledger_value): """ Rebuild a LedgerValue and its associated transactions during a form edit workflow. :param form_id: ID of edited form :param ledger_value: LedgerValue to rebuild with transactions from new form tracked on the model :return: updated LedgerValue object """ transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(**ledger_value.ledger_reference._asdict()) transaction_excluding_deprecated_form = [tx for tx in transactions if tx.form_id != form_id] new_transactions = ledger_value.get_tracked_models_to_create(LedgerTransaction) all_transactions = transaction_excluding_deprecated_form + new_transactions sorted_transactions = sorted(all_transactions, key=lambda t: t.report_date) ledger_value.clear_tracked_models(LedgerTransaction) ledger_value = self._rebuild_ledger_value_from_transactions(ledger_value, sorted_transactions) return ledger_value
def _diff_ledgers(self, case_ids): from corehq.apps.tzmigration.timezonemigration import json_diff from corehq.apps.commtrack.models import StockState couch_state_map = { state.ledger_reference: state for state in StockState.objects.filter(case_id__in=case_ids) } self.log_debug('Calculating ledger diffs for {} cases'.format(len(case_ids))) for ledger_value in LedgerAccessorSQL.get_ledger_values_for_cases(case_ids): couch_state = couch_state_map.get(ledger_value.ledger_reference, None) diffs = json_diff(couch_state.to_json(), ledger_value.to_json(), track_list_indices=False) self.diff_db.add_diffs( 'stock state', ledger_value.ledger_reference.as_id(), filter_ledger_diffs(diffs) )
def test_migrate_ledgers(self): case_id = uuid.uuid4().hex create_and_save_a_case(self.domain_name, case_id=case_id, case_name="Simon's sweet shop") self._set_balance(100, case_id, self.liquorice._id) self._set_balance(50, case_id, self.sherbert._id) self._set_balance(175, case_id, self.jelly_babies._id) expected_stock_state = {'stock': { self.liquorice._id: 100, self.sherbert._id: 50, self.jelly_babies._id: 175 }} self._validate_ledger_data(self._get_ledger_state(case_id), expected_stock_state) self._do_migration_and_assert_flags(self.domain_name) self._validate_ledger_data(self._get_ledger_state(case_id), expected_stock_state) transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id) self.assertEqual(3, len(transactions)) self._compare_diffs([])
def save_processed_models(cls, processed_forms, cases=None, stock_result=None): db_names = {processed_forms.submitted.db} if processed_forms.deprecated: db_names |= {processed_forms.deprecated.db} if cases: db_names |= {case.db for case in cases} if stock_result: db_names |= { ledger_value.db for ledger_value in stock_result.models_to_save } all_models = filter( None, chain( processed_forms, cases or [], stock_result.models_to_save if stock_result else [], )) try: with ExitStack() as stack: for db_name in db_names: stack.enter_context(transaction.atomic(db_name)) # Save deprecated form first to avoid ID conflicts if processed_forms.deprecated: FormAccessorSQL.update_form(processed_forms.deprecated, publish_changes=False) FormAccessorSQL.save_new_form(processed_forms.submitted) if cases: for case in cases: CaseAccessorSQL.save_case(case) if stock_result: ledgers_to_save = stock_result.models_to_save LedgerAccessorSQL.save_ledger_values( ledgers_to_save, stock_result) if cases: sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled( processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN) if sort_submissions: for case in cases: if SqlCaseUpdateStrategy( case).reconcile_transactions_if_necessary(): CaseAccessorSQL.save_case(case) except DatabaseError: for model in all_models: setattr(model, model._meta.pk.attname, None) for tracked in model.create_models: setattr(tracked, tracked._meta.pk.attname, None) raise try: cls.publish_changes_to_kafka(processed_forms, cases, stock_result) except Exception as e: raise KafkaPublishingError(e)
def _get_ledger(self, unique_ledger_reference): try: return LedgerAccessorSQL.get_ledger_value(**unique_ledger_reference._asdict()) except LedgerValueNotFound: return None
def get_ledgers_for_case(self, case_id): return LedgerAccessorSQL.get_ledger_values_for_case(case_id)
def reprocess_form(form, save=True, lock_form=True): if lock_form: # track load if locking; otherise it will be tracked elsewhere form_load_counter("reprocess_form", form.domain)() interface = FormProcessorInterface(form.domain) lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None with LockManager(form, lock): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' cache = interface.casedb_cache( domain=form.domain, lock=True, deleted_ok=True, xforms=[form], load_src="reprocess_form", ) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format(type(e).__name__, six.text_type(e)) form = interface.xformerror_from_xform_instance(form, error_message) return ReprocessingResult(form, [], [], error_message) form.initial_processing_complete = True form.problem = None stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases) ledgers = stock_result.models_to_save ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()} if save: for case in cases: CaseAccessorSQL.save_case(case) LedgerAccessorSQL.save_ledger_values(ledgers) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild updated ledgers interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict()) else: if save: interface.processor.save_processed_models([form], cases, stock_result) from casexml.apps.stock.models import StockTransaction ledgers = [ model for model in stock_result.models_to_save if isinstance(model, StockTransaction) ] for ledger in ledgers: interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict()) save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result) return ReprocessingResult(form, cases, ledgers, None)
def iter_all_changes(self, start_from=None): for ledger in LedgerAccessorSQL.get_all_ledgers_modified_since(start_from, chunk_size=self.chunk_size): yield _ledger_v2_to_change(ledger)
def _delete_ledgers_for_case(case_id): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id) form_ids = {tx.form_id for tx in transactions} for form_id in form_ids: LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id) LedgerAccessorSQL.delete_ledger_values(case_id)
def get_sql_ledger_values(case_ids): return LedgerAccessorSQL.get_ledger_values_for_cases(case_ids)
def test_delete_ledger_values_raise_error_case_section_entry(self): with self.assertRaisesRegexp(LedgerSaveError, '.*still has transactions.*'): LedgerAccessorSQL.delete_ledger_values(self.case.case_id, 'stock', self.product._id)
def get_tasks_case_immunization_ledger_values(tasks_case): if tasks_case.type != 'tasks': raise ValueError("Expected 'tasks' case") return LedgerAccessorSQL.get_ledger_values_for_cases([tasks_case.case_id], section_id='immuns')
def reprocess_form(form, save=True, lock_form=True): if lock_form: # track load if locking; otherise it will be tracked elsewhere form_load_counter("reprocess_form", form.domain)() interface = FormProcessorInterface(form.domain) lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None with LockManager(form, lock): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' cache = interface.casedb_cache( domain=form.domain, lock=True, deleted_ok=True, xforms=[form], load_src="reprocess_form", ) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format(type(e).__name__, str(e)) form = interface.xformerror_from_xform_instance(form, error_message) return ReprocessingResult(form, [], [], error_message) form.initial_processing_complete = True form.problem = None stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases) ledgers = stock_result.models_to_save ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()} if save: for case in cases: CaseAccessorSQL.save_case(case) LedgerAccessorSQL.save_ledger_values(ledgers) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild updated ledgers interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict()) else:
def get_current_ledger_value(self, unique_ledger_reference): try: return LedgerAccessorSQL.get_ledger_value(**unique_ledger_reference._asdict()).balance except LedgerValue.DoesNotExist: return 0
def test_delete_ledger_values_raise_error_case(self): with self.assertRaisesRegexp(LedgerSaveError, '.*still has transactions.*'): LedgerAccessorSQL.delete_ledger_values(self.case.case_id)
def get_sql_ledger_value(*args): return LedgerAccessorSQL.get_ledger_value(*args)
def test_reprocess_unfinished_submission_ledger_rebuild(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex form_ids = [] form_ids.append( submit_case_blocks([ CaseBlock(case_id=case_id, create=True, case_type='shop').as_string(), get_single_balance_block(case_id, 'product1', 100), ], self.domain)[0].form_id) transaction_patch = patch( 'corehq.form_processor.backends.sql.processor.transaction') ledger_save_patch = patch( 'corehq.form_processor.backends.sql.dbaccessors.LedgerAccessorSQL.save_ledger_values', side_effect=InternalError) with transaction_patch, ledger_save_patch, self.assertRaises( InternalError): submit_case_blocks( get_single_balance_block(case_id, 'product1', 50), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit another form afterwards form_ids.append( submit_case_blocks( get_single_balance_block(case_id, 'product1', 25), self.domain)[0].form_id) ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) self.assertEqual(25, ledgers[0].balance) ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case( case_id) self.assertEqual(2, len(ledger_transactions)) # should rebuild ledger transactions result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(0, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) # still only 1 self.assertEqual(25, ledgers[0].balance) ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case( case_id) self.assertEqual(3, len(ledger_transactions)) # make sure transactions are in correct order self.assertEqual(form_ids, [trans.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].updated_balance) self.assertEqual(100, ledger_transactions[0].delta) self.assertEqual(50, ledger_transactions[1].updated_balance) self.assertEqual(-50, ledger_transactions[1].delta) self.assertEqual(25, ledger_transactions[2].updated_balance) self.assertEqual(-25, ledger_transactions[2].delta)
def _reprocess_form(form, save=True): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' form.initial_processing_complete = True form.problem = None interface = FormProcessorInterface(form.domain) accessors = FormAccessors(form.domain) cache = interface.casedb_cache(domain=form.domain, lock=True, deleted_ok=True, xforms=[form]) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format(type(e).__name__, unicode(e)) form = interface.xformerror_from_xform_instance( form, error_message) accessors.update_form_problem_and_state(form) return ReprocessingResult(form, [], []) stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes('unfiltered', cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases = _filter_already_processed_cases(form, cases) cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases) if save: for case in cases: CaseAccessorSQL.save_case(case) ledgers = _filter_already_processed_ledgers( form, stock_result.models_to_save) ledgers_updated = { ledger.ledger_reference for ledger in ledgers if ledger.is_saved() } if save: LedgerAccessorSQL.save_ledger_values(ledgers) if save: FormAccessorSQL.update_form_problem_and_state(form) publish_form_saved(form) _log_changes('filtered', cases, ledgers, []) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) save and case_post_save.send(case.__class__, case=case) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild upated ledgers interface.ledger_processor.hard_rebuild_ledgers( **ledger.ledger_reference._asdict()) else:
def get_sql_transactions(case_id, section_id, entry_id): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case( case_id=case_id, section_id=section_id, entry_id=entry_id) return sorted(transactions, key=lambda t: (t.report_date, t.id))
def _get_ledger(self, unique_ledger_reference): try: return LedgerAccessorSQL.get_ledger_value( **unique_ledger_reference._asdict()) except LedgerValueNotFound: return None