コード例 #1
0
ファイル: ledger.py プロジェクト: kkrampa/commcare-hq
 def process_form_archived(self, form):
     from corehq.form_processor.parsers.ledgers.form import get_ledger_references_from_stock_transactions
     refs_to_rebuild = get_ledger_references_from_stock_transactions(form)
     case_ids = list({ref.case_id for ref in refs_to_rebuild})
     LedgerAccessorSQL.delete_ledger_transactions_for_form(case_ids, form.form_id)
     for ref in refs_to_rebuild:
         self.rebuild_ledger_state(**ref._asdict())
コード例 #2
0
def _create_ledger(domain, entry_id, balance, case_id=None, section_id='stock'):
    user_id = 'user1'
    utcnow = datetime.utcnow()

    case_id = case_id or uuid.uuid4().hex
    case = CommCareCaseSQL(
        case_id=case_id,
        domain=domain,
        type='',
        owner_id=user_id,
        opened_on=utcnow,
        modified_on=utcnow,
        modified_by=user_id,
        server_modified_on=utcnow,
    )

    CaseAccessorSQL.save_case(case)

    ledger = LedgerValue(
        domain=domain,
        case_id=case_id,
        section_id=section_id,
        entry_id=entry_id,
        balance=balance,
        last_modified=utcnow
    )

    LedgerAccessorSQL.save_ledger_values([ledger])
    return ledger
コード例 #3
0
ファイル: test_indicators.py プロジェクト: dimagi/commcare-hq
 def _create_models_for_stock_report_helper(self, form, stock_report_helper):
     processing_result = StockProcessingResult(form, stock_report_helpers=[stock_report_helper])
     processing_result.populate_models()
     if should_use_sql_backend(self.domain_name):
         from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
         LedgerAccessorSQL.save_ledger_values(processing_result.models_to_save)
     else:
         processing_result.commit()
コード例 #4
0
ファイル: utils.py プロジェクト: saketkanth/commcare-hq
 def delete_all_ledgers(domain):
     if should_use_sql_backend(domain):
         for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
             transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
             form_ids = {tx.form_id for tx in transactions}
             for form_id in form_ids:
                 LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
             LedgerAccessorSQL.delete_ledger_values(case_id)
     else:
         from casexml.apps.stock.models import StockReport
         from casexml.apps.stock.models import StockTransaction
         stock_report_ids = StockReport.objects.filter(domain=domain).values_list('id', flat=True)
         StockReport.objects.filter(domain=domain).delete()
         StockTransaction.objects.filter(report_id__in=stock_report_ids).delete()
コード例 #5
0
ファイル: ledger.py プロジェクト: dimagi/commcare-hq
    def process_form_unarchived(self, form):
        from corehq.apps.commtrack.processing import process_stock

        result = process_stock([form])
        result.populate_models()
        LedgerAccessorSQL.save_ledger_values(result.models_to_save)
        for ledger_value in result.models_to_save:
            publish_ledger_v2_saved(ledger_value)

        refs_to_rebuild = {ledger_value.ledger_reference for ledger_value in result.models_to_save}
        for ref in refs_to_rebuild:
            self.rebuild_ledger_state(**ref._asdict())

        result.finalize()
コード例 #6
0
def update_case_transactions_for_form(case_cache, live_case_updates, deprecated_case_updates,
                                      live_form, deprecated_form):
    for case_update in live_case_updates + deprecated_case_updates:
        case_id = case_update.id
        count, _ = CaseTransaction.objects.partitioned_query(case_id)\
            .filter(case_id=case_id, form_id=live_form.form_id).delete()

        rebuild_transactions = CaseTransaction.objects.partitioned_query(case_id).filter(
            case_id=case_id, type=CaseTransaction.TYPE_REBUILD_FORM_EDIT
        )
        for transaction in rebuild_transactions:
            if transaction.details.get('deprecated_form_id') == deprecated_form.original_form_id:
                transaction.delete()

    for case_update in live_case_updates:
        case_id = case_update.id
        case = case_cache.get(case_id)
        SqlCaseUpdateStrategy.add_transaction_for_form(case, case_update, live_form)

    for case_update in deprecated_case_updates:
        case_id = case_update.id
        case = case_cache.get(case_id)
        SqlCaseUpdateStrategy.add_transaction_for_form(case, case_update, deprecated_form)

    stock_result = process_stock([live_form, deprecated_form], case_cache)
    stock_result.populate_models()
    affected_ledgers = set()
    affected_cases = set()
    ledger_transactions = []
    for ledger_value in stock_result.models_to_save:
        affected_ledgers.add(ledger_value.ledger_reference)
        affected_cases.add(ledger_value.case_id)
        for transaction in ledger_value.get_tracked_models_to_create(LedgerTransaction):
            ledger_transactions.append(transaction)

    if affected_cases:
        LedgerAccessorSQL.delete_ledger_transactions_for_form(list(affected_cases), live_form.form_id)

    for transaction in ledger_transactions:
        transaction.save()

    for case in case_cache.cache.values():
        affected_cases.add(case.case_id)
        transactions = case.get_tracked_models_to_create(CaseTransaction)
        for transaction in transactions:
            transaction.case = case
            transaction.save()

    return affected_cases, affected_ledgers
コード例 #7
0
    def test_delete_ledger_values_case_section(self):
        from corehq.apps.commtrack.tests.util import get_single_balance_block
        form_id = self._submit_ledgers([
            get_single_balance_block(self.case_one.case_id, product_id, 10)
            for product_id in [self.product_a._id, self.product_b._id]
        ])
        ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id)
        self.assertEqual(2, len(ledger_values))

        LedgerAccessorSQL.delete_ledger_transactions_for_form([self.case_one.case_id], form_id)
        deleted = LedgerAccessorSQL.delete_ledger_values(self.case_one.case_id, 'stock')
        self.assertEqual(2, deleted)

        ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id)
        self.assertEqual(0, len(ledger_values))
コード例 #8
0
ファイル: ledger.py プロジェクト: saketkanth/commcare-hq
def _get_daily_consumption_for_ledger(ledger):
    from corehq.apps.commtrack.consumption import get_consumption_for_ledger_json
    daily_consumption = get_consumption_for_ledger_json(ledger)
    if should_use_sql_backend(ledger['domain']):
        from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
        ledger_value = LedgerAccessorSQL.get_ledger_value(
            ledger['case_id'], ledger['section_id'], ledger['entry_id']
        )
        ledger_value.daily_consumption = daily_consumption
        LedgerAccessorSQL.save_ledger_values([ledger_value])
    else:
        from corehq.apps.commtrack.models import StockState
        StockState.objects.filter(pk=ledger['_id']).update(daily_consumption=daily_consumption)

    return daily_consumption
コード例 #9
0
    def get_data(self):
        from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
        locations = self.locations()

        # locations at this point will only have location objects
        # that have supply points associated
        for loc in locations[:self.config.get('max_rows', 100)]:
            # TODO: this is very inefficient since it loads ALL the transactions up to the supplied
            # date but only requires the most recent one. Should rather use a window function.
            transactions = LedgerAccessorSQL.get_ledger_transactions_in_window(
                case_id=loc.supply_point_id,
                section_id=SECTION_TYPE_STOCK,
                entry_id=None,
                window_start=datetime.min,
                window_end=self.datetime,
            )

            if self.program_id:
                transactions = (
                    tx for tx in transactions
                    if tx.entry_id in self.product_ids
                )

            stock_results = sorted(transactions, key=lambda tx: tx.report_date, reverse=False)

            yield (loc.name, {tx.entry_id: tx.updated_balance for tx in stock_results})
コード例 #10
0
ファイル: ledger.py プロジェクト: ansarbek/commcare-hq
    def get_models_to_update(self, stock_report_helper, ledger_db=None):
        latest_values = {}
        for stock_trans in stock_report_helper.transactions:
            def _lazy_original_balance():
                # needs to be in closures because it's zero-argument.
                # see compute_ledger_values for more information
                if stock_trans.ledger_reference in latest_values:
                    return latest_values[stock_trans.ledger_reference]
                else:
                    return self.get_current_ledger_value(stock_trans.ledger_reference)

            new_ledger_values = compute_ledger_values(
                _lazy_original_balance, stock_report_helper.report_type, stock_trans.relative_quantity
            )
            latest_values[stock_trans.ledger_reference] = new_ledger_values.balance

        to_save = []
        for touched_ledger_reference, quantity in latest_values.items():
            try:
                ledger_value = LedgerAccessorSQL.get_ledger_value(**touched_ledger_reference._asdict())
            except LedgerValue.DoesNotExist:
                ledger_value = LedgerValue(**touched_ledger_reference._asdict())
            ledger_value.balance = quantity
            to_save.append(ledger_value)
        return StockModelUpdateResult(to_save=to_save)
コード例 #11
0
    def test_delete_ledger_values_case_section_1(self):
        from corehq.apps.commtrack.tests import get_single_balance_block
        form_id = self._submit_ledgers([
            get_single_balance_block(self.case_one.case_id, self.product_a._id, 10, section_id=section_id)
            for section_id in ['stock', 'consumption']
        ])
        ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id)
        self.assertEqual(2, len(ledger_values))

        LedgerAccessorSQL.delete_ledger_transactions_for_form([self.case_one.case_id], form_id)
        deleted = LedgerAccessorSQL.delete_ledger_values(self.case_one.case_id, 'stock')
        self.assertEqual(1, deleted)

        ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case_one.case_id)
        self.assertEqual(1, len(ledger_values))
        self.assertEqual('consumption', ledger_values[0].section_id)
コード例 #12
0
ファイル: processor.py プロジェクト: saketkanth/commcare-hq
    def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
        with transaction.atomic():
            logging.debug('Beginning atomic commit\n')
            # Save deprecated form first to avoid ID conflicts
            if processed_forms.deprecated:
                FormAccessorSQL.save_deprecated_form(processed_forms.deprecated)

            FormAccessorSQL.save_new_form(processed_forms.submitted)
            if cases:
                for case in cases:
                    CaseAccessorSQL.save_case(case)

            if stock_result:
                ledgers_to_save = stock_result.models_to_save
                LedgerAccessorSQL.save_ledger_values(ledgers_to_save, processed_forms.deprecated)

        cls._publish_changes(processed_forms, cases, stock_result)
コード例 #13
0
ファイル: processor.py プロジェクト: kkrampa/commcare-hq
    def save_processed_models(cls, processed_forms, cases=None, stock_result=None, publish_to_kafka=True):
        db_names = {processed_forms.submitted.db}
        if processed_forms.deprecated:
            db_names |= {processed_forms.deprecated.db}

        if cases:
            db_names |= {case.db for case in cases}

        if stock_result:
            db_names |= {
                ledger_value.db for ledger_value in stock_result.models_to_save
            }

        with ExitStack() as stack:
            for db_name in db_names:
                stack.enter_context(transaction.atomic(db_name))

            # Save deprecated form first to avoid ID conflicts
            if processed_forms.deprecated:
                FormAccessorSQL.update_form(processed_forms.deprecated, publish_changes=False)

            FormAccessorSQL.save_new_form(processed_forms.submitted)
            if cases:
                for case in cases:
                    CaseAccessorSQL.save_case(case)

            if stock_result:
                ledgers_to_save = stock_result.models_to_save
                LedgerAccessorSQL.save_ledger_values(ledgers_to_save, stock_result)

        if cases:
            sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled(
                processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN)
            if sort_submissions:
                for case in cases:
                    if SqlCaseUpdateStrategy(case).reconcile_transactions_if_necessary():
                        CaseAccessorSQL.save_case(case)

        if publish_to_kafka:
            try:
                cls.publish_changes_to_kafka(processed_forms, cases, stock_result)
            except Exception as e:
                raise KafkaPublishingError(e)
コード例 #14
0
    def setUp(self):
        # can't do this in setUpClass until Django 1.9 since @override_settings
        # doesn't apply to classmethods
        from corehq.apps.commtrack.tests import get_single_balance_block
        factory = CaseFactory(domain=self.domain)
        self.case = factory.create_case()
        submit_case_blocks([
            get_single_balance_block(self.case.case_id, self.product._id, 10)
        ], self.domain)

        ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(self.case.case_id)
        self.assertEqual(1, len(ledger_values))
コード例 #15
0
    def test_delete_ledger_transactions_for_form(self):
        from corehq.apps.commtrack.tests.util import get_single_balance_block

        self._set_balance(100, self.case_one.case_id, self.product_a._id)

        case_ids = [self.case_one.case_id, self.case_two.case_id]
        form_id = self._submit_ledgers([
            get_single_balance_block(case_id, product_id, 10)
            for case_id in case_ids
            for product_id in [self.product_a._id, self.product_b._id]
        ])
        deleted = LedgerAccessorSQL.delete_ledger_transactions_for_form(case_ids, form_id)
        self.assertEqual(4, deleted)
        self.assertEqual(
            1,
            len(LedgerAccessorSQL.get_ledger_transactions_for_case(self.case_one.case_id))
        )
        self.assertEqual(
            0,
            len(LedgerAccessorSQL.get_ledger_transactions_for_case(self.case_two.case_id))
        )
コード例 #16
0
 def _assert_transactions(self, values, ignore_ordering=False):
     if should_use_sql_backend(DOMAIN):
         txs = LedgerAccessorSQL.get_ledger_transactions_for_case(self.case.case_id)
         self.assertEqual(len(values), len(txs))
         if ignore_ordering:
             values = sorted(values, key=lambda v: (v.type, v.product_id))
             txs = sorted(txs, key=lambda t: (t.type, t.entry_id))
         for expected, tx in zip(values, txs):
             self.assertEqual(expected.type, tx.type)
             self.assertEqual(expected.product_id, tx.entry_id)
             self.assertEqual('stock', tx.section_id)
             self.assertEqual(expected.delta, tx.delta)
             self.assertEqual(expected.updated_balance, tx.updated_balance)
コード例 #17
0
    def test_ledgers(self):
        expected_object_counts = Counter({
            XFormInstanceSQL: 3,
            BlobMeta: 3,
            CommCareCaseSQL: 1,
            CaseTransaction: 3,
            LedgerValue: 1,
            LedgerTransaction: 2

        })

        case = self.factory.create_case()
        submit_case_blocks([
            get_single_balance_block(case.case_id, self.product._id, 10)
        ], self.domain_name)
        submit_case_blocks([
            get_single_balance_block(case.case_id, self.product._id, 5)
        ], self.domain_name)

        pre_ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(case.case_id)
        pre_ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case.case_id)
        self.assertEqual(1, len(pre_ledger_values))
        self.assertEqual(2, len(pre_ledger_transactions))

        self._dump_and_load(expected_object_counts)

        post_ledger_values = LedgerAccessorSQL.get_ledger_values_for_case(case.case_id)
        post_ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case.case_id)
        self.assertEqual(1, len(post_ledger_values))
        self.assertEqual(2, len(post_ledger_transactions))
        self.assertEqual(pre_ledger_values[0].ledger_reference, post_ledger_values[0].ledger_reference)
        self.assertDictEqual(pre_ledger_values[0].to_json(), post_ledger_values[0].to_json())

        pre_ledger_transactions = sorted(pre_ledger_transactions, key=lambda t: t.pk)
        post_ledger_transactions = sorted(post_ledger_transactions, key=lambda t: t.pk)
        for pre, post in zip(pre_ledger_transactions, post_ledger_transactions):
            self.assertEqual(str(pre), str(post))
コード例 #18
0
ファイル: ledger.py プロジェクト: dimagi/commcare-hq
 def hard_rebuild_ledgers(case_id, section_id=None, entry_id=None):
     transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id, section_id, entry_id)
     if not transactions:
         LedgerAccessorSQL.delete_ledger_values(case_id, section_id, entry_id)
         return
     ledger_value = LedgerAccessorSQL.get_ledger_value(case_id, section_id, entry_id)
     ledger_value = LedgerProcessorSQL._rebuild_ledger_value_from_transactions(ledger_value, transactions)
     LedgerAccessorSQL.save_ledger_values([ledger_value])
     publish_ledger_v2_saved(ledger_value)
コード例 #19
0
ファイル: ledger.py プロジェクト: dimagi/commcare-hq
    def _rebuild_ledger(self, form_id, ledger_value):
        """
        Rebuild a LedgerValue and its associated transactions during a form edit workflow.

        :param form_id: ID of edited form
        :param ledger_value: LedgerValue to rebuild with transactions from new form tracked on the model
        :return: updated LedgerValue object
        """
        transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(**ledger_value.ledger_reference._asdict())
        transaction_excluding_deprecated_form = [tx for tx in transactions if tx.form_id != form_id]
        new_transactions = ledger_value.get_tracked_models_to_create(LedgerTransaction)
        all_transactions = transaction_excluding_deprecated_form + new_transactions
        sorted_transactions = sorted(all_transactions, key=lambda t: t.report_date)

        ledger_value.clear_tracked_models(LedgerTransaction)
        ledger_value = self._rebuild_ledger_value_from_transactions(ledger_value, sorted_transactions)
        return ledger_value
コード例 #20
0
    def _diff_ledgers(self, case_ids):
        from corehq.apps.tzmigration.timezonemigration import json_diff
        from corehq.apps.commtrack.models import StockState
        couch_state_map = {
            state.ledger_reference: state
            for state in StockState.objects.filter(case_id__in=case_ids)
        }

        self.log_debug('Calculating ledger diffs for {} cases'.format(len(case_ids)))

        for ledger_value in LedgerAccessorSQL.get_ledger_values_for_cases(case_ids):
            couch_state = couch_state_map.get(ledger_value.ledger_reference, None)
            diffs = json_diff(couch_state.to_json(), ledger_value.to_json(), track_list_indices=False)
            self.diff_db.add_diffs(
                'stock state', ledger_value.ledger_reference.as_id(),
                filter_ledger_diffs(diffs)
            )
コード例 #21
0
    def test_migrate_ledgers(self):
        case_id = uuid.uuid4().hex
        create_and_save_a_case(self.domain_name, case_id=case_id, case_name="Simon's sweet shop")
        self._set_balance(100, case_id, self.liquorice._id)
        self._set_balance(50, case_id, self.sherbert._id)
        self._set_balance(175, case_id, self.jelly_babies._id)

        expected_stock_state = {'stock': {
            self.liquorice._id: 100,
            self.sherbert._id: 50,
            self.jelly_babies._id: 175
        }}
        self._validate_ledger_data(self._get_ledger_state(case_id), expected_stock_state)
        self._do_migration_and_assert_flags(self.domain_name)
        self._validate_ledger_data(self._get_ledger_state(case_id), expected_stock_state)

        transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
        self.assertEqual(3, len(transactions))

        self._compare_diffs([])
コード例 #22
0
ファイル: processor.py プロジェクト: fmagege/commcare-hq
    def save_processed_models(cls,
                              processed_forms,
                              cases=None,
                              stock_result=None):
        db_names = {processed_forms.submitted.db}
        if processed_forms.deprecated:
            db_names |= {processed_forms.deprecated.db}

        if cases:
            db_names |= {case.db for case in cases}

        if stock_result:
            db_names |= {
                ledger_value.db
                for ledger_value in stock_result.models_to_save
            }

        all_models = filter(
            None,
            chain(
                processed_forms,
                cases or [],
                stock_result.models_to_save if stock_result else [],
            ))
        try:
            with ExitStack() as stack:
                for db_name in db_names:
                    stack.enter_context(transaction.atomic(db_name))

                # Save deprecated form first to avoid ID conflicts
                if processed_forms.deprecated:
                    FormAccessorSQL.update_form(processed_forms.deprecated,
                                                publish_changes=False)

                FormAccessorSQL.save_new_form(processed_forms.submitted)
                if cases:
                    for case in cases:
                        CaseAccessorSQL.save_case(case)

                if stock_result:
                    ledgers_to_save = stock_result.models_to_save
                    LedgerAccessorSQL.save_ledger_values(
                        ledgers_to_save, stock_result)

            if cases:
                sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled(
                    processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN)
                if sort_submissions:
                    for case in cases:
                        if SqlCaseUpdateStrategy(
                                case).reconcile_transactions_if_necessary():
                            CaseAccessorSQL.save_case(case)
        except DatabaseError:
            for model in all_models:
                setattr(model, model._meta.pk.attname, None)
                for tracked in model.create_models:
                    setattr(tracked, tracked._meta.pk.attname, None)
            raise

        try:
            cls.publish_changes_to_kafka(processed_forms, cases, stock_result)
        except Exception as e:
            raise KafkaPublishingError(e)
コード例 #23
0
ファイル: ledger.py プロジェクト: kkrampa/commcare-hq
 def _get_ledger(self, unique_ledger_reference):
     try:
         return LedgerAccessorSQL.get_ledger_value(**unique_ledger_reference._asdict())
     except LedgerValueNotFound:
         return None
コード例 #24
0
ファイル: ledger.py プロジェクト: tstalka/commcare-hq
 def get_ledgers_for_case(self, case_id):
     return LedgerAccessorSQL.get_ledger_values_for_case(case_id)
コード例 #25
0
ファイル: reprocess.py プロジェクト: dimagi/commcare-hq
def reprocess_form(form, save=True, lock_form=True):
    if lock_form:
        # track load if locking; otherise it will be tracked elsewhere
        form_load_counter("reprocess_form", form.domain)()
    interface = FormProcessorInterface(form.domain)
    lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None
    with LockManager(form, lock):
        logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
        # reset form state prior to processing
        if should_use_sql_backend(form.domain):
            form.state = XFormInstanceSQL.NORMAL
        else:
            form.doc_type = 'XFormInstance'

        cache = interface.casedb_cache(
            domain=form.domain, lock=True, deleted_ok=True, xforms=[form],
            load_src="reprocess_form",
        )
        with cache as casedb:
            try:
                case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
            except (IllegalCaseId, UsesReferrals, MissingProductId,
                    PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
                error_message = '{}: {}'.format(type(e).__name__, six.text_type(e))
                form = interface.xformerror_from_xform_instance(form, error_message)
                return ReprocessingResult(form, [], [], error_message)

            form.initial_processing_complete = True
            form.problem = None

            stock_result = case_stock_result.stock_result
            assert stock_result.populated

            cases = case_stock_result.case_models
            _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete)

            ledgers = []
            if should_use_sql_backend(form.domain):
                cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)

                ledgers = stock_result.models_to_save
                ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()}

                if save:
                    for case in cases:
                        CaseAccessorSQL.save_case(case)
                    LedgerAccessorSQL.save_ledger_values(ledgers)
                    FormAccessorSQL.update_form_problem_and_state(form)
                    FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result)

                # rebuild cases and ledgers that were affected
                for case in cases:
                    if case.case_id in cases_needing_rebuild:
                        logger.info('Rebuilding case: %s', case.case_id)
                        if save:
                            # only rebuild cases that were updated
                            detail = FormReprocessRebuild(form_id=form.form_id)
                            interface.hard_rebuild_case(case.case_id, detail, lock=False)

                for ledger in ledgers:
                    if ledger.ledger_reference in ledgers_updated:
                        logger.info('Rebuilding ledger: %s', ledger.ledger_reference)
                        if save:
                            # only rebuild updated ledgers
                            interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())

            else:
                if save:
                    interface.processor.save_processed_models([form], cases, stock_result)
                    from casexml.apps.stock.models import StockTransaction
                    ledgers = [
                        model
                        for model in stock_result.models_to_save
                        if isinstance(model, StockTransaction)
                    ]
                    for ledger in ledgers:
                        interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())

            save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)

        return ReprocessingResult(form, cases, ledgers, None)
コード例 #26
0
 def iter_all_changes(self, start_from=None):
     for ledger in LedgerAccessorSQL.get_all_ledgers_modified_since(start_from, chunk_size=self.chunk_size):
         yield _ledger_v2_to_change(ledger)
コード例 #27
0
 def _delete_ledgers_for_case(case_id):
     transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
     form_ids = {tx.form_id for tx in transactions}
     for form_id in form_ids:
         LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
     LedgerAccessorSQL.delete_ledger_values(case_id)
コード例 #28
0
ファイル: retrydb.py プロジェクト: tobiasmcnulty/commcare-hq
def get_sql_ledger_values(case_ids):
    return LedgerAccessorSQL.get_ledger_values_for_cases(case_ids)
コード例 #29
0
 def test_delete_ledger_values_raise_error_case_section_entry(self):
     with self.assertRaisesRegexp(LedgerSaveError, '.*still has transactions.*'):
         LedgerAccessorSQL.delete_ledger_values(self.case.case_id, 'stock', self.product._id)
コード例 #30
0
def get_tasks_case_immunization_ledger_values(tasks_case):
    if tasks_case.type != 'tasks':
        raise ValueError("Expected 'tasks' case")

    return LedgerAccessorSQL.get_ledger_values_for_cases([tasks_case.case_id],
                                                         section_id='immuns')
コード例 #31
0
def reprocess_form(form, save=True, lock_form=True):
    if lock_form:
        # track load if locking; otherise it will be tracked elsewhere
        form_load_counter("reprocess_form", form.domain)()
    interface = FormProcessorInterface(form.domain)
    lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None
    with LockManager(form, lock):
        logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
        # reset form state prior to processing
        if should_use_sql_backend(form.domain):
            form.state = XFormInstanceSQL.NORMAL
        else:
            form.doc_type = 'XFormInstance'

        cache = interface.casedb_cache(
            domain=form.domain, lock=True, deleted_ok=True, xforms=[form],
            load_src="reprocess_form",
        )
        with cache as casedb:
            try:
                case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
            except (IllegalCaseId, UsesReferrals, MissingProductId,
                    PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
                error_message = '{}: {}'.format(type(e).__name__, str(e))
                form = interface.xformerror_from_xform_instance(form, error_message)
                return ReprocessingResult(form, [], [], error_message)

            form.initial_processing_complete = True
            form.problem = None

            stock_result = case_stock_result.stock_result
            assert stock_result.populated

            cases = case_stock_result.case_models
            _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete)

            ledgers = []
            if should_use_sql_backend(form.domain):
                cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)

                ledgers = stock_result.models_to_save
                ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()}

                if save:
                    for case in cases:
                        CaseAccessorSQL.save_case(case)
                    LedgerAccessorSQL.save_ledger_values(ledgers)
                    FormAccessorSQL.update_form_problem_and_state(form)
                    FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result)

                # rebuild cases and ledgers that were affected
                for case in cases:
                    if case.case_id in cases_needing_rebuild:
                        logger.info('Rebuilding case: %s', case.case_id)
                        if save:
                            # only rebuild cases that were updated
                            detail = FormReprocessRebuild(form_id=form.form_id)
                            interface.hard_rebuild_case(case.case_id, detail, lock=False)

                for ledger in ledgers:
                    if ledger.ledger_reference in ledgers_updated:
                        logger.info('Rebuilding ledger: %s', ledger.ledger_reference)
                        if save:
                            # only rebuild updated ledgers
                            interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())

            else:
コード例 #32
0
def get_tasks_case_immunization_ledger_values(tasks_case):
    if tasks_case.type != 'tasks':
        raise ValueError("Expected 'tasks' case")

    return LedgerAccessorSQL.get_ledger_values_for_cases([tasks_case.case_id], section_id='immuns')
コード例 #33
0
ファイル: ledger.py プロジェクト: ansarbek/commcare-hq
 def get_ledgers_for_case(self, case_id):
     return LedgerAccessorSQL.get_ledger_values_for_case(case_id)
コード例 #34
0
ファイル: ledger.py プロジェクト: ansarbek/commcare-hq
 def get_current_ledger_value(self, unique_ledger_reference):
     try:
         return LedgerAccessorSQL.get_ledger_value(**unique_ledger_reference._asdict()).balance
     except LedgerValue.DoesNotExist:
         return 0
コード例 #35
0
 def test_delete_ledger_values_raise_error_case(self):
     with self.assertRaisesRegexp(LedgerSaveError, '.*still has transactions.*'):
         LedgerAccessorSQL.delete_ledger_values(self.case.case_id)
コード例 #36
0
ファイル: retrydb.py プロジェクト: tobiasmcnulty/commcare-hq
def get_sql_ledger_value(*args):
    return LedgerAccessorSQL.get_ledger_value(*args)
コード例 #37
0
ファイル: utils.py プロジェクト: tstalka/commcare-hq
 def _delete_ledgers_for_case(case_id):
     transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
     form_ids = {tx.form_id for tx in transactions}
     for form_id in form_ids:
         LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
     LedgerAccessorSQL.delete_ledger_values(case_id)
コード例 #38
0
    def test_reprocess_unfinished_submission_ledger_rebuild(self):
        from corehq.apps.commtrack.tests.util import get_single_balance_block
        case_id = uuid.uuid4().hex
        form_ids = []
        form_ids.append(
            submit_case_blocks([
                CaseBlock(case_id=case_id, create=True,
                          case_type='shop').as_string(),
                get_single_balance_block(case_id, 'product1', 100),
            ], self.domain)[0].form_id)

        transaction_patch = patch(
            'corehq.form_processor.backends.sql.processor.transaction')
        ledger_save_patch = patch(
            'corehq.form_processor.backends.sql.dbaccessors.LedgerAccessorSQL.save_ledger_values',
            side_effect=InternalError)
        with transaction_patch, ledger_save_patch, self.assertRaises(
                InternalError):
            submit_case_blocks(
                get_single_balance_block(case_id, 'product1', 50), self.domain)

        stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
                                                        saved=False).all()
        self.assertEqual(1, len(stubs))
        form_ids.append(stubs[0].xform_id)

        # submit another form afterwards
        form_ids.append(
            submit_case_blocks(
                get_single_balance_block(case_id, 'product1', 25),
                self.domain)[0].form_id)

        ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id)
        self.assertEqual(1, len(ledgers))
        self.assertEqual(25, ledgers[0].balance)

        ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
            case_id)
        self.assertEqual(2, len(ledger_transactions))

        # should rebuild ledger transactions
        result = reprocess_unfinished_stub(stubs[0])
        self.assertEqual(0, len(result.cases))
        self.assertEqual(1, len(result.ledgers))

        ledgers = LedgerAccessorSQL.get_ledger_values_for_case(case_id)
        self.assertEqual(1, len(ledgers))  # still only 1
        self.assertEqual(25, ledgers[0].balance)

        ledger_transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
            case_id)
        self.assertEqual(3, len(ledger_transactions))
        # make sure transactions are in correct order
        self.assertEqual(form_ids,
                         [trans.form_id for trans in ledger_transactions])
        self.assertEqual(100, ledger_transactions[0].updated_balance)
        self.assertEqual(100, ledger_transactions[0].delta)
        self.assertEqual(50, ledger_transactions[1].updated_balance)
        self.assertEqual(-50, ledger_transactions[1].delta)
        self.assertEqual(25, ledger_transactions[2].updated_balance)
        self.assertEqual(-25, ledger_transactions[2].delta)
コード例 #39
0
ファイル: reprocess.py プロジェクト: knittingarch/commcare-hq
def _reprocess_form(form, save=True):
    logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
    # reset form state prior to processing
    if should_use_sql_backend(form.domain):
        form.state = XFormInstanceSQL.NORMAL
    else:
        form.doc_type = 'XFormInstance'

    form.initial_processing_complete = True
    form.problem = None

    interface = FormProcessorInterface(form.domain)
    accessors = FormAccessors(form.domain)
    cache = interface.casedb_cache(domain=form.domain,
                                   lock=True,
                                   deleted_ok=True,
                                   xforms=[form])
    with cache as casedb:
        try:
            case_stock_result = SubmissionPost.process_xforms_for_cases([form],
                                                                        casedb)
        except (IllegalCaseId, UsesReferrals, MissingProductId,
                PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
            error_message = '{}: {}'.format(type(e).__name__, unicode(e))
            form = interface.xformerror_from_xform_instance(
                form, error_message)
            accessors.update_form_problem_and_state(form)
            return ReprocessingResult(form, [], [])

        stock_result = case_stock_result.stock_result
        assert stock_result.populated

        cases = case_stock_result.case_models
        _log_changes('unfiltered', cases, stock_result.models_to_save,
                     stock_result.models_to_delete)

        ledgers = []
        if should_use_sql_backend(form.domain):
            cases = _filter_already_processed_cases(form, cases)
            cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)
            if save:
                for case in cases:
                    CaseAccessorSQL.save_case(case)

            ledgers = _filter_already_processed_ledgers(
                form, stock_result.models_to_save)
            ledgers_updated = {
                ledger.ledger_reference
                for ledger in ledgers if ledger.is_saved()
            }
            if save:
                LedgerAccessorSQL.save_ledger_values(ledgers)

            if save:
                FormAccessorSQL.update_form_problem_and_state(form)
                publish_form_saved(form)

            _log_changes('filtered', cases, ledgers, [])

            # rebuild cases and ledgers that were affected
            for case in cases:
                if case.case_id in cases_needing_rebuild:
                    logger.info('Rebuilding case: %s', case.case_id)
                    if save:
                        # only rebuild cases that were updated
                        detail = FormReprocessRebuild(form_id=form.form_id)
                        interface.hard_rebuild_case(case.case_id,
                                                    detail,
                                                    lock=False)
                save and case_post_save.send(case.__class__, case=case)

            for ledger in ledgers:
                if ledger.ledger_reference in ledgers_updated:
                    logger.info('Rebuilding ledger: %s',
                                ledger.ledger_reference)
                    if save:
                        # only rebuild upated ledgers
                        interface.ledger_processor.hard_rebuild_ledgers(
                            **ledger.ledger_reference._asdict())

        else:
コード例 #40
0
ファイル: ledgers.py プロジェクト: caktus/commcare-hq
def get_sql_transactions(case_id, section_id, entry_id):
    transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(
        case_id=case_id, section_id=section_id, entry_id=entry_id)
    return sorted(transactions, key=lambda t: (t.report_date, t.id))
コード例 #41
0
ファイル: ledger.py プロジェクト: tstalka/commcare-hq
 def _get_ledger(self, unique_ledger_reference):
     try:
         return LedgerAccessorSQL.get_ledger_value(
             **unique_ledger_reference._asdict())
     except LedgerValueNotFound:
         return None