def test_save_processed_models_deprecated(self): existing_form, new_form = _simulate_form_edit() FormProcessorSQL.save_processed_models( ProcessedForms(new_form, existing_form)) self._validate_deprecation(existing_form, new_form)
def _perfom_post_save_actions(form, save=True): interface = FormProcessorInterface(form.domain) cache = interface.casedb_cache( domain=form.domain, lock=False, deleted_ok=True, xforms=[form], load_src="reprocess_form_post_save", ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) case_models = case_stock_result.case_models if interface.use_sql_domain: forms = ProcessedForms(form, None) stock_result = case_stock_result.stock_result try: FormProcessorSQL.publish_changes_to_kafka(forms, case_models, stock_result) except Exception: error_message = "Error publishing to kafka" return ReprocessingResult(form, None, None, error_message) try: save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result) except PostSaveError: error_message = "Error performing post save operations" return ReprocessingResult(form, None, None, error_message) return ReprocessingResult(form, case_models, None, None)
class SqlUpdateStrategyTest(TestCase): DOMAIN = 'update-strategy-test-' + uuid.uuid4().hex USER_ID = 'mr_wednesday_' @classmethod def setUpClass(cls): super(SqlUpdateStrategyTest, cls).setUpClass() FormProcessorTestUtils.delete_all_sql_forms() FormProcessorTestUtils.delete_all_sql_cases() @classmethod def tearDownClass(cls): FormProcessorTestUtils.delete_all_sql_forms() FormProcessorTestUtils.delete_all_sql_cases() super(SqlUpdateStrategyTest, cls).tearDownClass() @patch.object(SoftAssert, '_call') def test_reconcile_transactions(self, soft_assert_mock): """ tests a transanction with an early client date and late server date """ with freeze_time("2018-10-10"): case = self._create_case() with freeze_time("2018-10-11"): new_old_xform = self._create_form() with freeze_time("2018-10-08"): new_old_trans = self._create_case_transaction(case, new_old_xform) with freeze_time("2018-10-11"): case.track_create(new_old_trans) FormProcessorSQL.save_processed_models( ProcessedForms(new_old_xform, []), [case])
def _save_migrated_models(sql_form, case_stock_result): """ See SubmissionPost.save_processed_models for ~what this should do. However, note that that function does some things that this one shouldn't, e.g. process ownership cleanliness flags. """ forms_tuple = ProcessedForms(sql_form, None) stock_result = case_stock_result.stock_result if case_stock_result else None if stock_result: assert stock_result.populated return FormProcessorSQL.save_processed_models( forms_tuple, cases=case_stock_result.case_models if case_stock_result else None, stock_result=stock_result, publish_to_kafka=False)
def create_case(case) -> CommCareCaseSQL: form = XFormInstanceSQL( form_id=uuid4().hex, xmlns='http://commcarehq.org/formdesigner/form-processor', received_on=case.server_modified_on, user_id=case.owner_id, domain=case.domain, ) transaction = CaseTransaction( type=CaseTransaction.TYPE_FORM, form_id=form.form_id, case=case, server_date=case.server_modified_on, ) with patch.object(FormProcessorSQL, "publish_changes_to_kafka"): case.track_create(transaction) processed_forms = ProcessedForms(form, []) FormProcessorSQL.save_processed_models(processed_forms, [case])
def _create_case(domain=None, form_id=None, case_type=None, user_id=None, closed=False, case_id=None): """ Create the models directly so that these tests aren't dependent on any other apps. Not testing form processing here anyway. :return: CommCareCaseSQL """ domain = domain or DOMAIN form_id = form_id or uuid.uuid4().hex case_id = case_id or uuid.uuid4().hex user_id = user_id or 'user1' utcnow = datetime.utcnow() form = XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=utcnow, user_id=user_id, domain=domain) case = CommCareCaseSQL(case_id=case_id, domain=domain, type=case_type or '', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, closed=closed or False) case.track_create(CaseTransaction.form_transaction(case, form, utcnow)) cases = [case] FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases) return CaseAccessorSQL.get_case(case_id)
if case_id: case = CommCareCaseSQL( case_id=case_id, domain=domain, type=case_type or '', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, closed=closed or False ) case.track_create(CaseTransaction.form_transaction(case, form)) cases = [case] FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases) return CaseAccessorSQL.get_case(case_id) def _create_case_with_index(referenced_case_id, identifier='parent', referenced_type='mother', relationship_id=CommCareCaseIndexSQL.CHILD, case_is_deleted=False, case_type='child'): case = _create_case(case_type=case_type) case.deleted = case_is_deleted index = CommCareCaseIndexSQL( case=case, identifier=identifier, referenced_type=referenced_type, referenced_id=referenced_case_id, relationship_id=relationship_id
modified_by=utcnow, server_modified_on=utcnow) form = self._create_form(user_id, utcnow) trans = self._create_case_transaction(case, form, utcnow, action_types=[128]) self._save(form, case, trans) return CaseAccessorSQL.get_case(case_id) def _save(self, form, case, transaction): case.track_create(transaction) FormProcessorSQL.save_processed_models( ProcessedForms(form, []), [case], # disable publish to Kafka to avoid intermittent errors caused by # the nexus of kafka's consumer thread and freeze_time publish_to_kafka=False, ) def _check_for_reconciliation_error_soft_assert(self, soft_assert_mock): for call in soft_assert_mock.call_args_list: self.assertNotIn('ReconciliationError', call[0][1]) soft_assert_mock.reset_mock() def test_update_known_properties_with_empty_values(): def test(prop): case = SqlCaseUpdateStrategy.case_implementation_class()
def reprocess_form(form, save=True, lock_form=True): interface = FormProcessorInterface(form.domain) lock = interface.acquire_lock_for_xform( form.form_id) if lock_form else None with LockManager(form, lock): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' cache = interface.casedb_cache(domain=form.domain, lock=True, deleted_ok=True, xforms=[form]) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases( [form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format( type(e).__name__, six.text_type(e)) form = interface.xformerror_from_xform_instance( form, error_message) return ReprocessingResult(form, [], [], error_message) form.initial_processing_complete = True form.problem = None stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases_needing_rebuild = _get_case_ids_needing_rebuild( form, cases) ledgers = stock_result.models_to_save ledgers_updated = { ledger.ledger_reference for ledger in ledgers if ledger.is_saved() } if save: for case in cases: CaseAccessorSQL.save_case(case) LedgerAccessorSQL.save_ledger_values(ledgers) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL._publish_changes( ProcessedForms(form, None), cases, stock_result) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild updated ledgers interface.ledger_processor.rebuild_ledger_state( **ledger.ledger_reference._asdict()) else:
def reprocess_xform_error(form): """ Attempt to re-process an error form. This was created specifically to address the issue of out of order forms and child cases (form creates child case before parent case has been created). See http://manage.dimagi.com/default.asp?250459 :param form_id: ID of the error form to process """ from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.submission_post import SubmissionPost from corehq.form_processor.utils import should_use_sql_backend from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL, FormAccessorSQL, LedgerAccessorSQL from corehq.blobs.mixin import bulk_atomic_blobs from couchforms.models import XFormInstance from casexml.apps.case.signals import case_post_save from corehq.form_processor.interfaces.processor import ProcessedForms from corehq.form_processor.backends.sql.processor import FormProcessorSQL if not form: raise Exception('Form with ID {} not found'.format(form.form_id)) if not form.is_error: raise Exception('Form was not an error form: {}={}'.format(form.form_id, form.doc_type)) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' form.initial_processing_complete = True form.problem = None cache = FormProcessorInterface(form.domain).casedb_cache( domain=form.domain, lock=True, deleted_ok=True, xforms=[form] ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) if case_stock_result: stock_result = case_stock_result.stock_result if stock_result: assert stock_result.populated cases = case_stock_result.case_models if should_use_sql_backend(form.domain): for case in cases: CaseAccessorSQL.save_case(case) if stock_result: LedgerAccessorSQL.save_ledger_values(stock_result.models_to_save) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL._publish_changes( ProcessedForms(form, None), cases, stock_result ) else: with bulk_atomic_blobs([form] + cases): XFormInstance.save(form) # use this save to that we don't overwrite the doc_type XFormInstance.get_db().bulk_save(cases) if stock_result: stock_result.commit() case_stock_result.stock_result.finalize() case_stock_result.case_result.commit_dirtiness_flags() for case in cases: case_post_save.send(case.__class__, case=case) return form
form = self._create_form(user_id, utcnow) trans = self._create_case_transaction(case, form, utcnow, action_types=[128]) self._save(form, case, trans) return CommCareCase.objects.get_case(case_id) def _save(self, form, case, transaction): # disable publish to Kafka to avoid intermittent errors caused by # the nexus of kafka's consumer thread and freeze_time with patch.object(FormProcessorSQL, "publish_changes_to_kafka"): case.track_create(transaction) FormProcessorSQL.save_processed_models(ProcessedForms(form, []), [case]) def _check_for_reconciliation_error_soft_assert(self, soft_assert_mock): for call in soft_assert_mock.call_args_list: self.assertNotIn('ReconciliationError', call[0][1]) soft_assert_mock.reset_mock() def test_update_known_properties_with_empty_values(): def test(prop): case = SqlCaseUpdateStrategy.case_implementation_class() setattr(case, prop, "value") action = CaseUpdateAction(block=None, **{prop: ""}) SqlCaseUpdateStrategy(case)._update_known_properties(action)
case = CaseAccessorSQL.get_case(case.case_id) update_strategy = SqlCaseUpdateStrategy(case) self.assertFalse(update_strategy.reconcile_transactions_if_necessary()) self._check_for_reconciliation_error_soft_assert(soft_assert_mock) def test_reconcile_not_necessary(self): with freeze_time("2018-10-10"): case = self._create_case() with freeze_time("2018-10-11"): new_old_xform = self._create_form() new_old_trans = self._create_case_transaction(case, new_old_xform) case.track_create(new_old_trans) FormProcessorSQL.save_processed_models( ProcessedForms(new_old_xform, []), [case]) case = CaseAccessorSQL.get_case(case.case_id) update_strategy = SqlCaseUpdateStrategy(case) self.assertFalse(update_strategy.reconcile_transactions_if_necessary()) def test_ignores_before_rebuild_transaction(self): with freeze_time("2018-10-10"): case = self._create_case() with freeze_time("2018-10-11"): new_old_xform = self._create_form() with freeze_time("2018-10-08"): new_old_trans = self._create_case_transaction(case, new_old_xform) with freeze_time("2018-10-11"): case.track_create(new_old_trans)