def test(self): form = make_form_from_case_blocks([ElementTree.fromstring(CASE_BLOCK)]) lock_manager = process_xform(form, domain=self.domain) with lock_manager as xforms: with CaseDbCache(domain=self.domain) as case_db: with self.assertRaises(PhoneDateValueError): process_cases_with_casedb(xforms, case_db)
def test(self): form = make_form_from_case_blocks([ElementTree.fromstring(CASE_BLOCK)]) result = process_xform_xml(self.domain, form) with result.get_locked_forms() as xforms: with self.interface.casedb_cache(domain=self.domain) as case_db: with self.assertRaises(PhoneDateValueError): process_cases_with_casedb(xforms, case_db)
def update_sync_log_with_checks(sync_log, xform, cases, case_db, case_id_blacklist=None): assert case_db is not None from casexml.apps.case.xform import CaseProcessingConfig case_id_blacklist = case_id_blacklist or [] try: sync_log.update_phone_lists(xform, cases) except SyncLogAssertionError as e: if e.case_id and e.case_id not in case_id_blacklist: form_ids = get_case_xform_ids(e.case_id) case_id_blacklist.append(e.case_id) for form_id in form_ids: if form_id != xform._id: form = XFormInstance.get(form_id) if form.doc_type == 'XFormInstance': from casexml.apps.case.xform import process_cases_with_casedb process_cases_with_casedb( [form], case_db, CaseProcessingConfig( strict_asserts=True, case_id_blacklist=case_id_blacklist ) ) updated_log = get_properly_wrapped_sync_log(sync_log._id) update_sync_log_with_checks(updated_log, xform, cases, case_db, case_id_blacklist=case_id_blacklist)
def test(self): form = _make_form_from_case_blocks([ElementTree.fromstring(CASE_BLOCK)]) result = process_xform_xml(self.domain, form) with result.get_locked_forms() as xforms: with self.interface.casedb_cache(domain=self.domain) as case_db: with self.assertRaises(PhoneDateValueError): process_cases_with_casedb(xforms, case_db)
def update_sync_log_with_checks(sync_log, xform, cases, case_db, case_id_blacklist=None): assert case_db is not None from casexml.apps.case.xform import CaseProcessingConfig case_id_blacklist = case_id_blacklist or [] try: sync_log.update_phone_lists(xform, cases) except SyncLogAssertionError, e: if e.case_id and e.case_id not in case_id_blacklist: form_ids = get_case_xform_ids(e.case_id) case_id_blacklist.append(e.case_id) for form_id in form_ids: if form_id != xform._id: form = XFormInstance.get(form_id) if form.doc_type == 'XFormInstance': from casexml.apps.case.xform import process_cases_with_casedb process_cases_with_casedb( [form], case_db, CaseProcessingConfig( strict_asserts=True, case_id_blacklist=case_id_blacklist ) ) updated_log = get_properly_wrapped_sync_log(sync_log._id) update_sync_log_with_checks(updated_log, xform, cases, case_db, case_id_blacklist=case_id_blacklist)
def test_casedb_already_has_cases(self): case = CaseFactory().create_case() case_db = CaseDbCache(initial=[CommCareCase(_id='fake1'), CommCareCase(_id='fake2')]) form = XFormInstance.get(case.xform_ids[0]) def assert_exactly_one_case(sender, xform, cases, **kwargs): global case_count case_count = len(cases) cases_received.connect(assert_exactly_one_case) try: process_cases_with_casedb([form], case_db) self.assertEqual(1, case_count) finally: cases_received.disconnect(assert_exactly_one_case)
def _process_cases(xform, config=None): """ Creates or updates case objects which live outside of the form. If reconcile is true it will perform an additional step of reconciling the case update history after the case is processed. """ warnings.warn("This function is deprecated. You should be using SubmissionPost.", DeprecationWarning) assert getattr(settings, "UNIT_TESTING", False) domain = get_and_check_xform_domain(xform) with CaseDbCacheCouch(domain=domain, lock=True, deleted_ok=True) as case_db: case_result = process_cases_with_casedb([xform], case_db, config=config) cases = case_result.cases docs = [xform] + cases now = datetime.datetime.utcnow() for case in cases: case.server_modified_on = now XFormInstance.get_db().bulk_save(docs) for case in cases: case_post_save.send(CommCareCase, case=case) case_result.commit_dirtiness_flags() return cases
def _process_cases(xform, config=None): """ Creates or updates case objects which live outside of the form. If reconcile is true it will perform an additional step of reconciling the case update history after the case is processed. """ warnings.warn( 'This function is deprecated. You should be using SubmissionPost.', DeprecationWarning, ) assert getattr(settings, 'UNIT_TESTING', False) domain = get_and_check_xform_domain(xform) with CaseDbCacheCouch(domain=domain, lock=True, deleted_ok=True) as case_db: case_result = process_cases_with_casedb([xform], case_db, config=config) cases = case_result.cases docs = [xform] + cases now = datetime.datetime.utcnow() for case in cases: case.server_modified_on = now
def test_casedb_already_has_cases(self): casedb_cache = FormProcessorInterface().casedb_cache case = CaseFactory().create_case() case_db = casedb_cache(initial=[CommCareCase(_id='fake1'), CommCareCase(_id='fake2')]) form = XFormInstance.get(case.xform_ids[0]) def assert_exactly_one_case(sender, xform, cases, **kwargs): global case_count case_count = len(cases) cases_received.connect(assert_exactly_one_case) try: process_cases_with_casedb([form], case_db) self.assertEqual(1, case_count) finally: cases_received.disconnect(assert_exactly_one_case)
def process_xforms_for_cases(xforms, case_db, timing_context=None): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock timing_context = timing_context or TimingContext() instance = xforms[0] with timing_context("process_cases"): case_result = process_cases_with_casedb(xforms, case_db) with timing_context("process_ledgers"): stock_result = process_stock(xforms, case_db) stock_result.populate_models() modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on with timing_context("check_cases_before_save"): cases = case_db.get_cases_for_saving(modified_on_date) return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
def reprocess_form_cases(form, config=None, case_db=None): """ For a given form, reprocess all case elements inside it. This operation should be a no-op if the form was sucessfully processed, but should correctly inject the update into the case history if the form was NOT successfully processed. """ from casexml.apps.case.xform import process_cases, process_cases_with_casedb if case_db: process_cases_with_casedb([form], case_db, config=config) else: process_cases(form, config) # mark cleaned up now that we've reprocessed it if form.doc_type != 'XFormInstance': form = XFormInstance.get(form._id) form.doc_type = 'XFormInstance' form.save()
def test_casedb_already_has_cases(self): casedb_cache = FormProcessorInterface().casedb_cache case = CaseFactory().create_case() case_db = casedb_cache(initial=[ CommCareCase(case_id='fake1'), CommCareCase(case_id='fake2'), ]) form = XFormInstance.objects.get_form(case.xform_ids[0]) received = [] def receive_cases(sender, xform, cases, **kwargs): received.extend(cases) cases_received.connect(receive_cases) try: process_cases_with_casedb([form], case_db) self.assertEqual(len(received), 1) finally: cases_received.disconnect(receive_cases)
def update_sync_log_with_checks(sync_log, xform, cases, case_db, case_id_blacklist=None): assert case_db is not None from casexml.apps.case.xform import CaseProcessingConfig case_id_blacklist = case_id_blacklist or [] try: sync_log.update_phone_lists(xform, cases) except SyncLogAssertionError as e: soft_assert('@'.join(['skelly', 'dimagi.com']))( False, 'SyncLogAssertionError raised while updating phone lists', { 'form_id': xform.form_id, 'cases': [case.case_id for case in cases] } ) if e.case_id and e.case_id not in case_id_blacklist: form_ids = get_case_xform_ids(e.case_id) case_id_blacklist.append(e.case_id) for form_id in form_ids: if form_id != xform._id: form = XFormInstance.get(form_id) if form.doc_type == 'XFormInstance': from casexml.apps.case.xform import process_cases_with_casedb process_cases_with_casedb( [form], case_db, CaseProcessingConfig( strict_asserts=True, case_id_blacklist=case_id_blacklist ) ) updated_log = get_properly_wrapped_sync_log(sync_log._id) update_sync_log_with_checks(updated_log, xform, cases, case_db, case_id_blacklist=case_id_blacklist)
def process_xforms_for_cases(xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(instance.received_on) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
def process_xforms_for_cases(self, xforms): from casexml.apps.case.xform import get_and_check_xform_domain from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] domain = get_and_check_xform_domain(instance) with self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db: case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(instance.received_on) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
def process_xforms_for_cases(xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on cases = case_db.get_cases_for_saving(modified_on_date) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
def run(self): if not self.auth_context.is_valid(): return self.failed_auth_response, None, [] if isinstance(self.instance, const.BadRequest): return HttpResponseBadRequest(self.instance.message), None, [] def process(xform): self._attach_shared_props(xform) scrub_meta(xform) try: lock_manager = process_xform(self.instance, attachments=self.attachments, process=process, domain=self.domain) except SubmissionError as e: logging.exception( u"Problem receiving submission to %s. %s" % ( self.path, unicode(e), ) ) return self.get_exception_response(e.error_log), None, [] else: from casexml.apps.case.models import CommCareCase from casexml.apps.case.xform import ( get_and_check_xform_domain, CaseDbCache, process_cases_with_casedb ) from casexml.apps.case.signals import case_post_save from casexml.apps.case.exceptions import IllegalCaseId, UsesReferrals from corehq.apps.commtrack.processing import process_stock cases = [] responses = [] errors = [] with lock_manager as xforms: instance = xforms[0] if instance.doc_type == 'XFormInstance': if len(xforms) > 1: assert len(xforms) == 2 assert is_deprecation(xforms[1]) domain = get_and_check_xform_domain(instance) with CaseDbCache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db: try: case_result = process_cases_with_casedb(xforms, case_db) process_stock(instance, case_db) except (IllegalCaseId, UsesReferrals) as e: # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form instance = _handle_known_error(e, instance) response = self._get_open_rosa_response(instance, None, None) return response, instance, cases except Exception as e: # handle / log the error and reraise so the phone knows to resubmit _handle_unexpected_error(e, instance) raise now = datetime.datetime.utcnow() unfinished_submission_stub = UnfinishedSubmissionStub( xform_id=instance.get_id, timestamp=now, saved=False, domain=domain, ) unfinished_submission_stub.save() cases = case_db.get_changed() # todo: this property is useless now instance.initial_processing_complete = True assert XFormInstance.get_db().uri == CommCareCase.get_db().uri docs = xforms + cases # in saving the cases, we have to do all the things # done in CommCareCase.save() for case in cases: case.initial_processing_complete = True case.server_modified_on = now try: rev = CommCareCase.get_db().get_rev(case.case_id) except ResourceNotFound: pass else: assert rev == case.get_rev, ( "Aborting because there would have been " "a document update conflict. {} {} {}".format( case.get_id, case.get_rev, rev ) ) try: # save both the forms and cases XFormInstance.get_db().bulk_save(docs) except BulkSaveError as e: logging.error('BulkSaveError saving forms', exc_info=1, extra={'details': {'errors': e.errors}}) raise unfinished_submission_stub.saved = True unfinished_submission_stub.save() for case in cases: case_post_save.send(CommCareCase, case=case) case_result.commit_dirtiness_flags() responses, errors = self.process_signals(instance) if errors: # .problems was added to instance instance.save() unfinished_submission_stub.delete() elif instance.doc_type == 'XFormDuplicate': assert len(xforms) == 1 instance.save() response = self._get_open_rosa_response(instance, responses, errors) return response, instance, cases
def run(self): if timezone_migration_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503), None, [] if not self.auth_context.is_valid(): return self.failed_auth_response, None, [] if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message), None, [] def process(xform): self._attach_shared_props(xform) if xform.doc_type != 'SubmissionErrorLog': found_old = scrub_meta(xform) legacy_soft_assert(not found_old, 'Form with old metadata submitted', xform._id) try: lock_manager = process_xform(self.instance, attachments=self.attachments, process=process, domain=self.domain) except SubmissionError as e: logging.exception( u"Problem receiving submission to %s. %s" % ( self.path, unicode(e), ) ) return self.get_exception_response(e.error_log), None, [] else: from casexml.apps.case.models import CommCareCase from casexml.apps.case.xform import ( get_and_check_xform_domain, CaseDbCache, process_cases_with_casedb ) from casexml.apps.case.signals import case_post_save from casexml.apps.case.exceptions import IllegalCaseId, UsesReferrals from corehq.apps.commtrack.processing import process_stock from corehq.apps.commtrack.exceptions import MissingProductId cases = [] responses = [] errors = [] known_errors = (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError) with lock_manager as xforms: instance = xforms[0] if instance.doc_type == 'XFormInstance': if len(xforms) > 1: assert len(xforms) == 2 assert is_deprecation(xforms[1]) domain = get_and_check_xform_domain(instance) with CaseDbCache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db: try: case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) except known_errors as e: # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form instance = _handle_known_error(e, instance) xforms[0] = instance # this is usually just one document, but if an edit errored we want # to save the deprecated form as well XFormInstance.get_db().bulk_save(xforms) response = self._get_open_rosa_response( instance, None) return response, instance, cases except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" error_message = u'{}: {}'.format(type(e).__name__, unicode(e)) instance = _handle_unexpected_error(instance, error_message) instance.save() raise now = datetime.datetime.utcnow() unfinished_submission_stub = UnfinishedSubmissionStub( xform_id=instance.get_id, timestamp=now, saved=False, domain=domain, ) unfinished_submission_stub.save() cases = case_db.get_changed() # todo: this property is only used by the MVPFormIndicatorPillow instance.initial_processing_complete = True # in saving the cases, we have to do all the things # done in CommCareCase.save() for case in cases: legacy_soft_assert(case.version == "2.0", "v1.0 case updated", case.case_id) case.initial_processing_complete = True case.server_modified_on = now try: rev = CommCareCase.get_db().get_rev(case.case_id) except ResourceNotFound: pass else: assert rev == case.get_rev, ( "Aborting because there would have been " "a document update conflict. {} {} {}".format( case.get_id, case.get_rev, rev ) ) # verify that these DB's are the same so that we can save them with one call to bulk_save assert XFormInstance.get_db().uri == CommCareCase.get_db().uri docs = xforms + cases try: XFormInstance.get_db().bulk_save(docs) except BulkSaveError as e: logging.error('BulkSaveError saving forms', exc_info=1, extra={'details': {'errors': e.errors}}) raise except Exception as e: docs_being_saved = [doc['_id'] for doc in docs] error_message = u'Unexpected error bulk saving docs {}: {}, doc_ids: {}'.format( type(e).__name__, unicode(e), ', '.join(docs_being_saved) ) instance = _handle_unexpected_error(instance, error_message) instance.save() raise unfinished_submission_stub.saved = True unfinished_submission_stub.save() case_result.commit_dirtiness_flags() stock_result.commit() for case in cases: case_post_save.send(CommCareCase, case=case) errors = self.process_signals(instance) if errors: # .problems was added to instance instance.save() unfinished_submission_stub.delete() elif instance.doc_type == 'XFormDuplicate': assert len(xforms) == 1 instance.save() response = self._get_open_rosa_response(instance, errors) return response, instance, cases
def run(self): if not self.auth_context.is_valid(): return self.failed_auth_response, None, [] if isinstance(self.instance, const.BadRequest): return HttpResponseBadRequest(self.instance.message), None, [] def process(xform): self._attach_shared_props(xform) scrub_meta(xform) try: lock_manager = process_xform(self.instance, attachments=self.attachments, process=process, domain=self.domain) except SubmissionError as e: logging.exception( u"Problem receiving submission to %s. %s" % ( self.path, unicode(e), ) ) return self.get_exception_response(e.error_log), None, [] else: from casexml.apps.case.models import CommCareCase from casexml.apps.case.xform import ( get_and_check_xform_domain, CaseDbCache, process_cases_with_casedb ) from casexml.apps.case.signals import case_post_save from casexml.apps.case.exceptions import IllegalCaseId, UsesReferrals from corehq.apps.commtrack.processing import process_stock from corehq.apps.commtrack.exceptions import MissingProductId cases = [] responses = [] errors = [] known_errors = (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError) with lock_manager as xforms: instance = xforms[0] if instance.doc_type == 'XFormInstance': if len(xforms) > 1: assert len(xforms) == 2 assert is_deprecation(xforms[1]) domain = get_and_check_xform_domain(instance) with CaseDbCache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db: try: case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(instance, case_db) except known_errors as e: # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form instance = _handle_known_error(e, instance) xforms[0] = instance # this is usually just one document, but if an edit errored we want # to save the deprecated form as well XFormInstance.get_db().bulk_save(xforms) response = self._get_open_rosa_response(instance, None, None) return response, instance, cases except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" error_message = u'{}: {}'.format(type(e).__name__, unicode(e)) instance = _handle_unexpected_error(instance, error_message) instance.save() raise now = datetime.datetime.utcnow() unfinished_submission_stub = UnfinishedSubmissionStub( xform_id=instance.get_id, timestamp=now, saved=False, domain=domain, ) unfinished_submission_stub.save() cases = case_db.get_changed() # todo: this property is useless now instance.initial_processing_complete = True assert XFormInstance.get_db().uri == CommCareCase.get_db().uri docs = xforms + cases # in saving the cases, we have to do all the things # done in CommCareCase.save() for case in cases: case.initial_processing_complete = True case.server_modified_on = now try: rev = CommCareCase.get_db().get_rev(case.case_id) except ResourceNotFound: pass else: assert rev == case.get_rev, ( "Aborting because there would have been " "a document update conflict. {} {} {}".format( case.get_id, case.get_rev, rev ) ) try: # save both the forms and cases XFormInstance.get_db().bulk_save(docs) except BulkSaveError as e: logging.error('BulkSaveError saving forms', exc_info=1, extra={'details': {'errors': e.errors}}) raise except Exception as e: docs_being_saved = [doc['_id'] for doc in docs] error_message = u'Unexpected error bulk saving docs {}: {}, doc_ids: {}'.format( type(e).__name__, unicode(e), ', '.join(docs_being_saved) ) instance = _handle_unexpected_error(instance, error_message) instance.save() raise unfinished_submission_stub.saved = True unfinished_submission_stub.save() case_result.commit_dirtiness_flags() stock_result.commit() for case in cases: case_post_save.send(CommCareCase, case=case) responses, errors = self.process_signals(instance) if errors: # .problems was added to instance instance.save() unfinished_submission_stub.delete() elif instance.doc_type == 'XFormDuplicate': assert len(xforms) == 1 instance.save()
soft_assert('@'.join(['skelly', 'dimagi.com']))( False, 'SyncLogAssertionError raised while updating phone lists', { 'form_id': xform.form_id, 'cases': [case.case_id for case in cases] }) if e.case_id and e.case_id not in case_id_blacklist: form_ids = get_case_xform_ids(e.case_id) case_id_blacklist.append(e.case_id) for form_id in form_ids: if form_id != xform._id: form = XFormInstance.get(form_id) if form.doc_type == 'XFormInstance': from casexml.apps.case.xform import process_cases_with_casedb process_cases_with_casedb( [form], case_db, CaseProcessingConfig( strict_asserts=True, case_id_blacklist=case_id_blacklist)) updated_log = get_properly_wrapped_sync_log(sync_log._id) update_sync_log_with_checks(updated_log, xform, cases, case_db, case_id_blacklist=case_id_blacklist) def get_indexed_cases(domain, case_ids): """ Given a base list of cases, gets all wrapped cases that they reference (parent cases).