class CaseDbCacheCouchOnlyTest(TestCase): def setUp(self): super(CaseDbCacheCouchOnlyTest, self).setUp() self.interface = FormProcessorInterface() def testDocTypeCheck(self): id = uuid.uuid4().hex CommCareCase.get_db().save_doc({ "_id": id, "doc_type": "AintNoCasesHere" }) doc_back = CommCareCase.get_db().get(id) self.assertEqual("AintNoCasesHere", doc_back['doc_type']) cache = CaseDbCacheCouch() try: cache.get(id) self.fail('doc type security check failed to raise exception') except IllegalCaseId: pass def testStripHistory(self): case_ids = _make_some_cases(3) history_cache = self.interface.casedb_cache() for i, id in enumerate(case_ids): self.assertFalse(history_cache.in_cache(id)) case = history_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) > 0) nohistory_cache = self.interface.casedb_cache(strip_history=True) for i, id in enumerate(case_ids): self.assertFalse(nohistory_cache.in_cache(id)) case = nohistory_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) == 0) more_case_ids = _make_some_cases(3) history_cache.populate(more_case_ids) nohistory_cache.populate(more_case_ids) for i, id in enumerate(more_case_ids): self.assertTrue(history_cache.in_cache(id)) case = history_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) > 0) for i, id in enumerate(more_case_ids): self.assertTrue(nohistory_cache.in_cache(id)) case = nohistory_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) == 0) def test_nowrap(self): case_ids = _make_some_cases(1) cache = self.interface.casedb_cache(wrap=False) case = cache.get(case_ids[0]) self.assertTrue(isinstance(case, dict)) self.assertFalse(isinstance(case, CommCareCase))
class CaseDbCacheTest(TestCase): """ Tests the functionality of the CaseDbCache object """ def setUp(self): super(CaseDbCacheTest, self).setUp() self.interface = FormProcessorInterface() @run_with_all_backends def testDomainCheck(self): id = uuid.uuid4().hex post_case_blocks( [CaseBlock(create=True, case_id=id, user_id='some-user').as_xml()], {'domain': 'good-domain'}) bad_cache = self.interface.casedb_cache(domain='bad-domain') try: bad_cache.get(id) self.fail('domain security check failed to raise exception') except IllegalCaseId: pass good_cache = self.interface.casedb_cache(domain='good-domain') case = good_cache.get(id) self.assertEqual( 'some-user', case.user_id) # just sanity check it's the right thing def testDocTypeCheck(self): id = uuid.uuid4().hex CommCareCase.get_db().save_doc({ "_id": id, "doc_type": "AintNoCasesHere" }) doc_back = CommCareCase.get_db().get(id) self.assertEqual("AintNoCasesHere", doc_back['doc_type']) cache = CaseDbCacheCouch() try: cache.get(id) self.fail('doc type security check failed to raise exception') except IllegalCaseId: pass @run_with_all_backends def testGetPopulatesCache(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) for i, id in enumerate(case_ids): case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) for id in case_ids: self.assertTrue(cache.in_cache(id))
def _perfom_post_save_actions(form, save=True): interface = FormProcessorInterface(form.domain) cache = interface.casedb_cache( domain=form.domain, lock=False, deleted_ok=True, xforms=[form], load_src="reprocess_form_post_save", ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) case_models = case_stock_result.case_models if interface.use_sql_domain: forms = ProcessedForms(form, None) stock_result = case_stock_result.stock_result try: FormProcessorSQL.publish_changes_to_kafka(forms, case_models, stock_result) except Exception: error_message = "Error publishing to kafka" return ReprocessingResult(form, None, None, error_message) try: save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result) except PostSaveError: error_message = "Error performing post save operations" return ReprocessingResult(form, None, None, error_message) return ReprocessingResult(form, case_models, None, None)
class CaseDbCacheCouchOnlyTest(TestCase): def setUp(self): super(CaseDbCacheCouchOnlyTest, self).setUp() self.interface = FormProcessorInterface() def testDocTypeCheck(self): id = uuid.uuid4().hex CommCareCase.get_db().save_doc({ "_id": id, "doc_type": "AintNoCasesHere" }) doc_back = CommCareCase.get_db().get(id) self.assertEqual("AintNoCasesHere", doc_back['doc_type']) cache = CaseDbCacheCouch() try: cache.get(id) self.fail('doc type security check failed to raise exception') except IllegalCaseId: pass def test_nowrap(self): case_ids = _make_some_cases(1) cache = self.interface.casedb_cache(wrap=False) case = cache.get(case_ids[0]) self.assertTrue(isinstance(case, dict)) self.assertFalse(isinstance(case, CommCareCase))
def _perfom_post_save_actions(form, save=True): interface = FormProcessorInterface(form.domain) cache = interface.casedb_cache( domain=form.domain, lock=False, deleted_ok=True, xforms=[form] ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) try: save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result) except PostSaveError: error_message = "Error performing post save operations" return ReprocessingResult(form, None, None, error_message) return ReprocessingResult(form, case_stock_result.case_models, None, None)
class CaseDbCacheTest(TestCase): """ Tests the functionality of the CaseDbCache object """ def setUp(self): super(CaseDbCacheTest, self).setUp() self.interface = FormProcessorInterface() def testDomainCheck(self): id = uuid.uuid4().hex post_case_blocks([ CaseBlock.deprecated_init( create=True, case_id=id, user_id='some-user').as_xml() ], {'domain': 'good-domain'}) bad_cache = self.interface.casedb_cache(domain='bad-domain') try: bad_cache.get(id) self.fail('domain security check failed to raise exception') except IllegalCaseId: pass good_cache = self.interface.casedb_cache(domain='good-domain') case = good_cache.get(id) self.assertEqual( 'some-user', case.user_id) # just sanity check it's the right thing def testGetPopulatesCache(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) for i, id in enumerate(case_ids): case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) for id in case_ids: self.assertTrue(cache.in_cache(id))
def _get_case_and_ledger_updates(domain, sql_form): """ Get a CaseStockProcessingResult with the appropriate cases and ledgers to be saved. See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent section of the form-processing code. """ from corehq.apps.commtrack.processing import process_stock interface = FormProcessorInterface(domain) assert sql_form.domain xforms = [sql_form] with interface.casedb_cache( domain=domain, lock=False, deleted_ok=True, xforms=xforms, load_src="couchsqlmigration", ) as case_db: touched_cases = FormProcessorInterface(domain).get_cases_from_forms( case_db, xforms) extensions_to_close = get_all_extensions_to_close( domain, list(touched_cases.values())) case_result = CaseProcessingResult( domain, [update.case for update in touched_cases.values()], [], # ignore dirtiness_flags, extensions_to_close) for case in case_result.cases: case_db.post_process_case(case, sql_form) case_db.mark_changed(case) cases = case_result.cases try: stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(sql_form.received_on) stock_result.populate_models() except MissingFormXml: stock_result = None return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
def _get_case_and_ledger_updates(domain, sql_form): """ Get a CaseStockProcessingResult with the appropriate cases and ledgers to be saved. See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent section of the form-processing code. """ from casexml.apps.case.xform import get_and_check_xform_domain from corehq.apps.commtrack.processing import process_stock interface = FormProcessorInterface(domain) get_and_check_xform_domain(sql_form) xforms = [sql_form] # todo: I think this can be changed to lock=False with interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db: touched_cases = FormProcessorInterface(domain).get_cases_from_forms( case_db, xforms) extensions_to_close = get_all_extensions_to_close( domain, touched_cases.values()) case_result = CaseProcessingResult( domain, [update.case for update in touched_cases.values()], [], # ignore dirtiness_flags, extensions_to_close) # todo: is this necessary? for case in case_result.cases: case_db.mark_changed(case) stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(sql_form.received_on) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
def _get_case_and_ledger_updates(domain, sql_form): """ Get a CaseStockProcessingResult with the appropriate cases and ledgers to be saved. See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent section of the form-processing code. """ from corehq.apps.commtrack.processing import process_stock interface = FormProcessorInterface(domain) assert sql_form.domain xforms = [sql_form] with interface.casedb_cache( domain=domain, lock=False, deleted_ok=True, xforms=xforms, load_src="couchsqlmigration", ) as case_db: touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms) extensions_to_close = get_all_extensions_to_close(domain, list(touched_cases.values())) case_result = CaseProcessingResult( domain, [update.case for update in touched_cases.values()], [], # ignore dirtiness_flags, extensions_to_close ) for case in case_result.cases: case_db.post_process_case(case, sql_form) case_db.mark_changed(case) stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(sql_form.received_on) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
class SubmissionPost(object): def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.auth_context = self.auth_context.to_json() xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form instance = _transform_instance_to_error(self.interface, error, instance) xforms[0] = instance # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if any_migrations_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503) if not self.auth_context.is_valid(): return HttpResponseForbidden('Bad auth') if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message) def _post_process_form(self, xform): self._set_submission_properties(xform) found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def run(self): failure_response = self._handle_basic_failure_modes() if failure_response: return FormProcessingResult(failure_response, None, [], [], 'known_failures') result = process_xform_xml(self.domain, self.instance, self.attachments) submitted_form = result.submitted_form self._post_process_form(submitted_form) self._invalidate_caches(submitted_form.user_id) submission_type = None if submitted_form.is_submission_error_log: self.formdb.save_new_form(submitted_form) response = self.get_exception_response_and_log(submitted_form, self.path) return FormProcessingResult(response, None, [], [], 'submission_error_log') cases = [] ledgers = [] submission_type = 'unknown' with result.get_locked_forms() as xforms: from casexml.apps.case.xform import get_and_check_xform_domain domain = get_and_check_xform_domain(xforms[0]) if self.case_db: assert self.case_db.domain == domain case_db_cache = self.case_db case_db_cache.cached_xforms.extend(xforms) else: case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) with case_db_cache as case_db: instance = xforms[0] if instance.xmlns == DEVICE_LOG_XMLNS: submission_type = 'device_log' try: process_device_log(self.domain, instance) except Exception: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) raise elif instance.is_duplicate: submission_type = 'duplicate' self.interface.save_processed_models([instance]) elif not instance.is_error: submission_type = 'normal' try: case_stock_result = self.process_xforms_for_cases(xforms, case_db) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: self._handle_known_error(e, instance, xforms) submission_type = 'error' except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True self.save_processed_models(xforms, case_stock_result) case_stock_result.case_result.close_extensions(case_db) cases = case_stock_result.case_models ledgers = case_stock_result.stock_result.models_to_save elif instance.is_error: submission_type = 'error' errors = self.process_signals(instance) response = self._get_open_rosa_response(instance, errors) return FormProcessingResult(response, instance, cases, ledgers, submission_type) @property def _cache(self): return get_redis_default_cache() @property def _restore_cache_key(self): from casexml.apps.phone.restore import restore_cache_key return restore_cache_key def _invalidate_caches(self, user_id): """invalidate cached initial restores""" initial_restore_cache_key = self._restore_cache_key( self.domain, RESTORE_CACHE_KEY_PREFIX, user_id, version=V2 ) self._cache.delete(initial_restore_cache_key) if ASYNC_RESTORE.enabled(self.domain): self._invalidate_async_caches(user_id) def _invalidate_async_caches(self, user_id): cache_key = self._restore_cache_key(self.domain, ASYNC_RESTORE_CACHE_KEY_PREFIX, user_id) task_id = self._cache.get(cache_key) if task_id is not None: revoke_celery_task(task_id) self._cache.delete(cache_key) def save_processed_models(self, xforms, case_stock_result): from casexml.apps.case.signals import case_post_save instance = xforms[0] with unfinished_submission(instance) as unfinished_submission_stub: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result ) unfinished_submission_stub.saved = True unfinished_submission_stub.save() case_stock_result.case_result.commit_dirtiness_flags() case_stock_result.stock_result.finalize() for case in case_stock_result.case_models: case_post_save.send(case.__class__, case=case) @staticmethod def process_xforms_for_cases(xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on cases = case_db.get_cases_for_saving(modified_on_date) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): return self.run().response def process_signals(self, instance): feedback = successful_form_received.send_robust(None, xform=instance) errors = [] for func, resp in feedback: if resp and isinstance(resp, Exception): error_message = unicode(resp) logging.error(( u"Receiver app: problem sending " u"post-save signal %s for xform %s: %s: %s" ) % (func, instance.form_id, type(resp).__name__, error_message)) errors.append(error_message) if errors: self.interface.xformerror_from_xform_instance(instance, ", ".join(errors)) self.formdb.update_form_problem_and_state(instance) return errors def _get_open_rosa_response(self, instance, errors): if instance.is_normal and not errors: response = self.get_success_response() else: response = self.get_failure_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_success_response(): return OpenRosaResponse( # would have done ✓ but our test Nokias' fonts don't have that character message=u' √ ', nature=ResponseNature.SUBMIT_SUCCESS, status=201, ).response() @staticmethod def submission_ignored_response(): return OpenRosaResponse( # would have done ✓ but our test Nokias' fonts don't have that character message=u'√ (this submission was ignored)', nature=ResponseNature.SUBMIT_SUCCESS, status=201, ).response() @staticmethod def get_failure_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_exception_response_and_log(error_instance, path): logging.exception( u"Problem receiving submission to %s. Doc id: %s, Error %s" % ( path, error_instance.form_id, error_instance.problem ) ) return OpenRosaResponse( message="There was an error processing the form: %s" % error_instance.problem, nature=ResponseNature.SUBMIT_ERROR, status=500, ).response() @staticmethod def get_blacklisted_response(): return OpenRosaResponse( message=("This submission was blocked because of an unusual volume " "of submissions from this project space. Please contact " "support to resolve."), nature=ResponseNature.SUBMIT_ERROR, status=509, ).response()
def reprocess_form(form, save=True, lock_form=True): interface = FormProcessorInterface(form.domain) lock = interface.acquire_lock_for_xform( form.form_id) if lock_form else None with LockManager(form, lock): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' cache = interface.casedb_cache(domain=form.domain, lock=True, deleted_ok=True, xforms=[form]) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases( [form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format( type(e).__name__, six.text_type(e)) form = interface.xformerror_from_xform_instance( form, error_message) return ReprocessingResult(form, [], [], error_message) form.initial_processing_complete = True form.problem = None stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases_needing_rebuild = _get_case_ids_needing_rebuild( form, cases) ledgers = stock_result.models_to_save ledgers_updated = { ledger.ledger_reference for ledger in ledgers if ledger.is_saved() } if save: for case in cases: CaseAccessorSQL.save_case(case) LedgerAccessorSQL.save_ledger_values(ledgers) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL._publish_changes( ProcessedForms(form, None), cases, stock_result) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild updated ledgers interface.ledger_processor.rebuild_ledger_state( **ledger.ledger_reference._asdict()) else:
class CaseDbCacheTest(TestCase): """ Tests the functionality of the CaseDbCache object """ def setUp(self): super(CaseDbCacheTest, self).setUp() self.interface = FormProcessorInterface() def testDomainCheck(self): id = uuid.uuid4().hex post_case_blocks([ CaseBlock( create=True, case_id=id, user_id='some-user' ).as_xml() ], {'domain': 'good-domain'} ) bad_cache = self.interface.casedb_cache(domain='bad-domain') try: bad_cache.get(id) self.fail('domain security check failed to raise exception') except IllegalCaseId: pass good_cache = self.interface.casedb_cache(domain='good-domain') case = good_cache.get(id) self.assertEqual('some-user', case.user_id) # just sanity check it's the right thing def testGetPopulatesCache(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) for i, id in enumerate(case_ids): case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) for id in case_ids: self.assertTrue(cache.in_cache(id)) def testSetPopulatesCache(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) for id in case_ids: cache.set(id, CaseAccessors().get_case(id)) for i, id in enumerate(case_ids): self.assertTrue(cache.in_cache(id)) case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) def testPopulate(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) cache.populate(case_ids) for id in case_ids: self.assertTrue(cache.in_cache(id)) # sanity check for i, id in enumerate(case_ids): case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index'])
class SubmissionPost(object): def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form xforms[0] = _transform_instance_to_error(self.interface, error, instance) # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if any_migrations_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503) if not self.auth_context.is_valid(): return HttpResponseForbidden('Bad auth') if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message) def _post_process_form(self, xform): self._set_submission_properties(xform) found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def _get_success_message(self, instance, cases=None): ''' Formplayer requests get a detailed success message pointing to the form/case affected. All other requests get a generic message. Message is formatted with markdown. ''' if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID: return ' √ ' messages = [] user = CouchUser.get_by_user_id(instance.user_id) if not user or not user.is_web_user(): return _('Form successfully saved!') from corehq.apps.export.views import CaseExportListView, FormExportListView from corehq.apps.reports.views import CaseDataView, FormDataView form_link = case_link = form_export_link = case_export_link = None form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory' if has_permission_to_view_report(user, instance.domain, form_view): form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id]) case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport' if cases and has_permission_to_view_report(user, instance.domain, case_view): if len(cases) == 1: case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id]) else: case_link = ", ".join(["[{}]({})".format( c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id]) ) for c in cases]) if can_view_form_exports(user, instance.domain): form_export_link = reverse(FormExportListView.urlname, args=[instance.domain]) if cases and can_view_case_exports(user, instance.domain): case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain]) # Start with generic message messages.append(_('Form successfully saved!')) # Add link to form/case if possible if form_link and case_link: if len(cases) == 1: messages.append( _("You submitted [this form]({}), which affected [this case]({}).") .format(form_link, case_link)) else: messages.append( _("You submitted [this form]({}), which affected these cases: {}.") .format(form_link, case_link)) elif form_link: messages.append(_("You submitted [this form]({}).").format(form_link)) elif case_link: if len(cases) == 1: messages.append(_("Your form affected [this case]({}).").format(case_link)) else: messages.append(_("Your form affected these cases: {}.").format(case_link)) # Add link to all form/case exports if form_export_link and case_export_link: messages.append( _("Click to export your [case]({}) or [form]({}) data.") .format(case_export_link, form_export_link)) elif form_export_link: messages.append(_("Click to export your [form data]({}).").format(form_export_link)) elif case_export_link: messages.append(_("Click to export your [case data]({}).").format(case_export_link)) return "\n\n".join(messages) def run(self): failure_response = self._handle_basic_failure_modes() if failure_response: return FormProcessingResult(failure_response, None, [], [], 'known_failures') result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json()) submitted_form = result.submitted_form self._post_process_form(submitted_form) self._invalidate_caches(submitted_form) submission_type = None if submitted_form.is_submission_error_log: self.formdb.save_new_form(submitted_form) response = self.get_exception_response_and_log(submitted_form, self.path) return FormProcessingResult(response, None, [], [], 'submission_error_log') cases = [] ledgers = [] submission_type = 'unknown' openrosa_kwargs = {} with result.get_locked_forms() as xforms: from casexml.apps.case.xform import get_and_check_xform_domain domain = get_and_check_xform_domain(xforms[0]) if self.case_db: assert self.case_db.domain == domain case_db_cache = self.case_db case_db_cache.cached_xforms.extend(xforms) else: case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) with case_db_cache as case_db: instance = xforms[0] if instance.xmlns == DEVICE_LOG_XMLNS: submission_type = 'device_log' self._conditionally_send_device_logs_to_sumologic(instance) # ignore temporarily till we migrate DeviceReportEntry id to bigint ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.ICDS_ENVS if not ignore_device_logs and instance.xmlns == DEVICE_LOG_XMLNS: submission_type = 'device_log' try: process_device_log(self.domain, instance) except Exception as e: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) e.sentry_capture = False raise elif instance.is_duplicate: submission_type = 'duplicate' existing_form = xforms[1] stub = UnfinishedSubmissionStub.objects.filter( domain=instance.domain, xform_id=existing_form.form_id ).first() result = None if stub: from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False) elif existing_form.is_error: from corehq.form_processor.reprocess import reprocess_form result = reprocess_form(existing_form, lock_form=False) if result and result.error: submission_type = 'error' openrosa_kwargs['error_message'] = result.error if existing_form.is_error: openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE else: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE else: self.interface.save_processed_models([instance]) elif not instance.is_error: submission_type = 'normal' try: case_stock_result = self.process_xforms_for_cases(xforms, case_db) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: self._handle_known_error(e, instance, xforms) submission_type = 'error' openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms, case_stock_result) if openrosa_kwargs['error_message']: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE cases = case_stock_result.case_models ledgers = case_stock_result.stock_result.models_to_save openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases) elif instance.is_error: submission_type = 'error' response = self._get_open_rosa_response(instance, **openrosa_kwargs) return FormProcessingResult(response, instance, cases, ledgers, submission_type) def _conditionally_send_device_logs_to_sumologic(self, instance): url = getattr(settings, 'SUMOLOGIC_URL', None) if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER): send_device_logs_to_sumologic.delay(self.domain, instance, url) def _invalidate_caches(self, xform): for device_id in {None, xform.metadata.deviceID if xform.metadata else None}: self._invalidate_restore_payload_path_cache(xform, device_id) if ASYNC_RESTORE.enabled(self.domain): self._invalidate_async_restore_task_id_cache(xform, device_id) def _invalidate_restore_payload_path_cache(self, xform, device_id): """invalidate cached initial restores""" restore_payload_path_cache = RestorePayloadPathCache( domain=self.domain, user_id=xform.user_id, sync_log_id=xform.last_sync_token, device_id=device_id, ) restore_payload_path_cache.invalidate() def _invalidate_async_restore_task_id_cache(self, xform, device_id): async_restore_task_id_cache = AsyncRestoreTaskIdCache( domain=self.domain, user_id=xform.user_id, sync_log_id=self.last_sync_token, device_id=device_id, ) task_id = async_restore_task_id_cache.get_value() if task_id is not None: revoke_celery_task(task_id) async_restore_task_id_cache.invalidate() def save_processed_models(self, case_db, xforms, case_stock_result): instance = xforms[0] try: with unfinished_submission(instance) as unfinished_submission_stub: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result ) if unfinished_submission_stub: unfinished_submission_stub.saved = True unfinished_submission_stub.save() self.do_post_save_actions(case_db, xforms, case_stock_result) except PostSaveError: return "Error performing post save operations" @staticmethod def do_post_save_actions(case_db, xforms, case_stock_result): instance = xforms[0] try: case_stock_result.case_result.commit_dirtiness_flags() case_stock_result.stock_result.finalize() SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models) case_stock_result.case_result.close_extensions( case_db, "SubmissionPost-%s-close_extensions" % instance.form_id ) except PostSaveError: raise except Exception: notify_exception(get_request(), "Error performing post save actions during form processing", { 'domain': instance.domain, 'form_id': instance.form_id, }) raise PostSaveError @staticmethod def process_xforms_for_cases(xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on cases = case_db.get_cases_for_saving(modified_on_date) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): return self.run().response @staticmethod def _fire_post_save_signals(instance, cases): from casexml.apps.case.signals import case_post_save error_message = "Error occurred during form submission post save (%s)" error_details = {'domain': instance.domain, 'form_id': instance.form_id} results = successful_form_received.send_robust(None, xform=instance) has_errors = log_signal_errors(results, error_message, error_details) for case in cases: results = case_post_save.send_robust(case.__class__, case=case) has_errors |= log_signal_errors(results, error_message, error_details) if has_errors: raise PostSaveError def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None): if self.is_openrosa_version3: instance_ok = instance.is_normal or instance.is_duplicate has_error = error_message or error_nature if instance_ok and not has_error: response = openrosa_response.get_openarosa_success_response(message=success_message) else: error_message = error_message or instance.problem response = self.get_retry_response(error_message, error_nature) else: if instance.is_normal: response = openrosa_response.get_openarosa_success_response() else: response = self.get_v2_submit_error_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_v2_submit_error_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_retry_response(message, nature): """Returns a 422(Unprocessable Entity) response, mobile will retry this submission """ return OpenRosaResponse( message=message, nature=nature, status=422, ).response() @staticmethod def get_exception_response_and_log(error_instance, path): logging.exception( "Problem receiving submission to %s. Doc id: %s, Error %s" % ( path, error_instance.form_id, error_instance.problem ) ) return OpenRosaResponse( message="There was an error processing the form: %s" % error_instance.problem, nature=ResponseNature.SUBMIT_ERROR, status=500, ).response()
def reprocess_form(form, save=True, lock_form=True): if lock_form: # track load if locking; otherise it will be tracked elsewhere form_load_counter("reprocess_form", form.domain)() interface = FormProcessorInterface(form.domain) lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None with LockManager(form, lock): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' cache = interface.casedb_cache( domain=form.domain, lock=True, deleted_ok=True, xforms=[form], load_src="reprocess_form", ) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format(type(e).__name__, six.text_type(e)) form = interface.xformerror_from_xform_instance(form, error_message) return ReprocessingResult(form, [], [], error_message) form.initial_processing_complete = True form.problem = None stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases) ledgers = stock_result.models_to_save ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()} if save: for case in cases: CaseAccessorSQL.save_case(case) LedgerAccessorSQL.save_ledger_values(ledgers) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild updated ledgers interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict()) else: if save: interface.processor.save_processed_models([form], cases, stock_result) from casexml.apps.stock.models import StockTransaction ledgers = [ model for model in stock_result.models_to_save if isinstance(model, StockTransaction) ] for ledger in ledgers: interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict()) save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result) return ReprocessingResult(form, cases, ledgers, None)
class SubmissionPost(object): def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None, force_logs=False, timing_context=None): assert domain, "'domain' is required" assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db if case_db: assert case_db.domain == domain self.force_logs = force_logs self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 self.track_load = form_load_counter("form_submission", domain) self.timing_context = timing_context or TimingContext() def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form xforms[0] = _transform_instance_to_error(self.interface, error, instance) # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if any_migrations_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503) if not self.auth_context.is_valid(): return HttpResponseForbidden('Bad auth') def _post_process_form(self, xform): self._set_submission_properties(xform) found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def _get_success_message(self, instance, cases=None): ''' Formplayer requests get a detailed success message pointing to the form/case affected. All other requests get a generic message. Message is formatted with markdown. ''' if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID: return ' √ ' messages = [] user = CouchUser.get_by_user_id(instance.user_id) if not user or not user.is_web_user(): return _('Form successfully saved!') from corehq.apps.export.views.list import CaseExportListView, FormExportListView from corehq.apps.export.views.utils import can_view_case_exports, can_view_form_exports from corehq.apps.reports.views import CaseDataView, FormDataView form_link = case_link = form_export_link = case_export_link = None form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory' if has_permission_to_view_report(user, instance.domain, form_view): form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id]) case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport' if cases and has_permission_to_view_report(user, instance.domain, case_view): if len(cases) == 1: case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id]) else: case_link = ", ".join(["[{}]({})".format( c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id]) ) for c in cases]) if can_view_form_exports(user, instance.domain): form_export_link = reverse(FormExportListView.urlname, args=[instance.domain]) if cases and can_view_case_exports(user, instance.domain): case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain]) # Start with generic message messages.append(_('Form successfully saved!')) # Add link to form/case if possible if form_link and case_link: if len(cases) == 1: messages.append( _("You submitted [this form]({}), which affected [this case]({}).") .format(form_link, case_link)) else: messages.append( _("You submitted [this form]({}), which affected these cases: {}.") .format(form_link, case_link)) elif form_link: messages.append(_("You submitted [this form]({}).").format(form_link)) elif case_link: if len(cases) == 1: messages.append(_("Your form affected [this case]({}).").format(case_link)) else: messages.append(_("Your form affected these cases: {}.").format(case_link)) # Add link to all form/case exports if form_export_link and case_export_link: messages.append( _("Click to export your [case]({}) or [form]({}) data.") .format(case_export_link, form_export_link)) elif form_export_link: messages.append(_("Click to export your [form data]({}).").format(form_export_link)) elif case_export_link: messages.append(_("Click to export your [case data]({}).").format(case_export_link)) return "\n\n".join(messages) def run(self): self.track_load() with self.timing_context("process_xml"): report_submission_usage(self.domain) failure_response = self._handle_basic_failure_modes() if failure_response: return FormProcessingResult(failure_response, None, [], [], 'known_failures') result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json()) submitted_form = result.submitted_form self._post_process_form(submitted_form) self._invalidate_caches(submitted_form) if submitted_form.is_submission_error_log: logging.info('Processing form %s as a submission error', submitted_form.form_id) XFormInstance.objects.save_new_form(submitted_form) response = None try: xml = self.instance.decode() except UnicodeDecodeError: pass else: if 'log_subreport' in xml: response = self.get_exception_response_and_log( 'Badly formed device log', submitted_form, self.path ) if not response: response = self.get_exception_response_and_log( 'Problem receiving submission', submitted_form, self.path ) return FormProcessingResult(response, None, [], [], 'submission_error_log') if submitted_form.xmlns == SYSTEM_ACTION_XMLNS: logging.info('Processing form %s as a system action', submitted_form.form_id) with self.timing_context("process_system_action"): return self.handle_system_action(submitted_form) if submitted_form.xmlns == DEVICE_LOG_XMLNS: logging.info('Processing form %s as a device log', submitted_form.form_id) with self.timing_context("process_device_log"): return self.process_device_log(submitted_form) # Begin Normal Form Processing self._log_form_details(submitted_form) cases = [] ledgers = [] submission_type = 'unknown' openrosa_kwargs = {} with result.get_locked_forms() as xforms: if len(xforms) > 1: self.track_load(len(xforms) - 1) if self.case_db: case_db_cache = self.case_db case_db_cache.cached_xforms.extend(xforms) else: case_db_cache = self.interface.casedb_cache( domain=self.domain, lock=True, deleted_ok=True, xforms=xforms, load_src="form_submission", ) with case_db_cache as case_db: instance = xforms[0] if instance.is_duplicate: with self.timing_context("process_duplicate"), tracer.trace('submission.process_duplicate'): submission_type = 'duplicate' existing_form = xforms[1] stub = UnfinishedSubmissionStub.objects.filter( domain=instance.domain, xform_id=existing_form.form_id ).first() result = None if stub: from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False) elif existing_form.is_error: from corehq.form_processor.reprocess import reprocess_form result = reprocess_form(existing_form, lock_form=False) if result and result.error: submission_type = 'error' openrosa_kwargs['error_message'] = result.error if existing_form.is_error: openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE else: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE else: self.interface.save_processed_models([instance]) elif not instance.is_error: submission_type = 'normal' try: case_stock_result = self.process_xforms_for_cases(xforms, case_db, self.timing_context) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: self._handle_known_error(e, instance, xforms) submission_type = 'error' openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms, case_stock_result) if openrosa_kwargs['error_message']: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE cases = case_stock_result.case_models ledgers = case_stock_result.stock_result.models_to_save report_case_usage(self.domain, len(cases)) openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases) elif instance.is_error: submission_type = 'error' self._log_form_completion(instance, submission_type) response = self._get_open_rosa_response(instance, **openrosa_kwargs) return FormProcessingResult(response, instance, cases, ledgers, submission_type) def _log_form_details(self, form): attachments = form.attachments if hasattr(form, 'attachments') else {} logging.info('Received Form %s with %d attachments', form.form_id, len(attachments)) for index, (name, attachment) in enumerate(attachments.items()): attachment_msg = 'Form %s, Attachment %s: %s' attachment_props = [form.form_id, index, name] if hasattr(attachment, 'has_size') and attachment.has_size(): attachment_msg = attachment_msg + ' (%d bytes)' attachment_props.append(attachment.raw_content.size) logging.info(attachment_msg, *attachment_props) def _log_form_completion(self, form, submission_type): # Orig_id doesn't exist on all couch forms, only XFormError and XFormDeprecated if hasattr(form, 'orig_id') and form.orig_id is not None: logging.info('Finished %s processing for Form %s with original id %s', submission_type, form.form_id, form.orig_id) else: logging.info('Finished %s processing for Form %s', submission_type, form.form_id) def _conditionally_send_device_logs_to_sumologic(self, instance): url = getattr(settings, 'SUMOLOGIC_URL', None) if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER): SumoLogicLog(self.domain, instance).send_data(url) def _invalidate_caches(self, xform): for device_id in {None, xform.metadata.deviceID if xform.metadata else None}: self._invalidate_restore_payload_path_cache(xform, device_id) if ASYNC_RESTORE.enabled(self.domain): self._invalidate_async_restore_task_id_cache(xform, device_id) def _invalidate_restore_payload_path_cache(self, xform, device_id): """invalidate cached initial restores""" restore_payload_path_cache = RestorePayloadPathCache( domain=self.domain, user_id=xform.user_id, sync_log_id=xform.last_sync_token, device_id=device_id, ) restore_payload_path_cache.invalidate() def _invalidate_async_restore_task_id_cache(self, xform, device_id): async_restore_task_id_cache = AsyncRestoreTaskIdCache( domain=self.domain, user_id=xform.user_id, sync_log_id=self.last_sync_token, device_id=device_id, ) task_id = async_restore_task_id_cache.get_value() if task_id is not None: revoke_celery_task(task_id) async_restore_task_id_cache.invalidate() @tracer.wrap(name='submission.save_models') def save_processed_models(self, case_db, xforms, case_stock_result): instance = xforms[0] try: with self.timing_context("save_models"), unfinished_submission(instance) as unfinished_submission_stub: try: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result ) except PostSaveError: # mark the stub as saved if there's a post save error # but re-raise the error so that the re-processing queue picks it up unfinished_submission_stub.submission_saved() raise else: unfinished_submission_stub.submission_saved() with self.timing_context("post_save_actions"): self.do_post_save_actions(case_db, xforms, case_stock_result) except PostSaveError: return "Error performing post save operations" @staticmethod @tracer.wrap(name='submission.post_save_actions') def do_post_save_actions(case_db, xforms, case_stock_result): instance = xforms[0] case_db.clear_changed() try: case_stock_result.stock_result.finalize() SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models) close_extension_cases( case_db, case_stock_result.case_models, "SubmissionPost-%s-close_extensions" % instance.form_id ) except PostSaveError: raise except Exception: notify_exception(get_request(), "Error performing post save actions during form processing", { 'domain': instance.domain, 'form_id': instance.form_id, }) raise PostSaveError @staticmethod @tracer.wrap(name='submission.process_cases_and_stock') def process_xforms_for_cases(xforms, case_db, timing_context=None): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock timing_context = timing_context or TimingContext() instance = xforms[0] with timing_context("process_cases"): case_result = process_cases_with_casedb(xforms, case_db) with timing_context("process_ledgers"): stock_result = process_stock(xforms, case_db) stock_result.populate_models() modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on with timing_context("check_cases_before_save"): cases = case_db.get_cases_for_saving(modified_on_date) return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): return self.run().response @staticmethod def _fire_post_save_signals(instance, cases): from corehq.form_processor.signals import sql_case_post_save error_message = "Error occurred during form submission post save (%s)" error_details = {'domain': instance.domain, 'form_id': instance.form_id} results = successful_form_received.send_robust(None, xform=instance) has_errors = log_signal_errors(results, error_message, error_details) for case in cases: results = sql_case_post_save.send_robust(case.__class__, case=case) has_errors |= log_signal_errors(results, error_message, error_details) if has_errors: raise PostSaveError def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None): if self.is_openrosa_version3: instance_ok = instance.is_normal or instance.is_duplicate has_error = error_message or error_nature if instance_ok and not has_error: response = openrosa_response.get_openarosa_success_response(message=success_message) else: error_message = error_message or instance.problem response = self.get_v3_error_response(error_message, error_nature) else: if instance.is_normal: response = openrosa_response.get_openarosa_success_response() else: response = self.get_v2_submit_error_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_v2_submit_error_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_v3_error_response(message, nature): """Returns a 422(Unprocessable Entity) response - if nature == 'processing_failure' the mobile device will quarantine this form and not retry it - any other value of `nature` will result in the form being marked as a failure and retrying """ return OpenRosaResponse( message=message, nature=nature, status=422, ).response() @staticmethod def get_exception_response_and_log(msg, error_instance, path): logging.warning( msg, extra={ 'submission_path': path, 'form_id': error_instance.form_id, 'error_message': error_instance.problem } ) # This are generally badly formed XML resulting from file corruption, encryption errors # or other errors on the device which can not be recovered from. # To prevent retries of these errors we submit a 422 response with `processing_failure` nature. return OpenRosaResponse( message="There was an error processing the form: %s" % error_instance.problem, nature=ResponseNature.PROCESSING_FAILURE, status=422, ).response() @tracer.wrap(name='submission.handle_system_action') def handle_system_action(self, form): handle_system_action(form, self.auth_context) self.interface.save_processed_models([form]) response = HttpResponse(status=201) return FormProcessingResult(response, form, [], [], 'system-action') @tracer.wrap(name='submission.process_device_log') def process_device_log(self, device_log_form): self._conditionally_send_device_logs_to_sumologic(device_log_form) ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.NO_DEVICE_LOG_ENVS if self.force_logs or not ignore_device_logs: try: process_device_log(self.domain, device_log_form, self.force_logs) except Exception as e: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) e.sentry_capture = False raise response = self._get_open_rosa_response(device_log_form) return FormProcessingResult(response, device_log_form, [], [], 'device-log')
class CaseDbCacheTest(TestCase): """ Tests the functionality of the CaseDbCache object """ def setUp(self): self.interface = FormProcessorInterface() @run_with_all_backends def testDomainCheck(self): id = uuid.uuid4().hex post_case_blocks([ CaseBlock( create=True, case_id=id, user_id='some-user' ).as_xml() ], {'domain': 'good-domain'} ) bad_cache = self.interface.casedb_cache(domain='bad-domain') try: bad_cache.get(id) self.fail('domain security check failed to raise exception') except IllegalCaseId: pass good_cache = self.interface.casedb_cache(domain='good-domain') case = good_cache.get(id) self.assertEqual('some-user', case.user_id) # just sanity check it's the right thing def testDocTypeCheck(self): id = uuid.uuid4().hex CommCareCase.get_db().save_doc({ "_id": id, "doc_type": "AintNoCasesHere" }) doc_back = CommCareCase.get_db().get(id) self.assertEqual("AintNoCasesHere", doc_back['doc_type']) cache = CaseDbCacheCouch() try: cache.get(id) self.fail('doc type security check failed to raise exception') except IllegalCaseId: pass @run_with_all_backends def testGetPopulatesCache(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) for i, id in enumerate(case_ids): case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) for id in case_ids: self.assertTrue(cache.in_cache(id)) @run_with_all_backends def testSetPopulatesCache(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) for id in case_ids: cache.set(id, CaseAccessors().get_case(id)) for i, id in enumerate(case_ids): self.assertTrue(cache.in_cache(id)) case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) @run_with_all_backends def testPopulate(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) cache.populate(case_ids) for id in case_ids: self.assertTrue(cache.in_cache(id)) # sanity check for i, id in enumerate(case_ids): case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) def testStripHistory(self): case_ids = _make_some_cases(3) history_cache = self.interface.casedb_cache() for i, id in enumerate(case_ids): self.assertFalse(history_cache.in_cache(id)) case = history_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) > 0) nohistory_cache = self.interface.casedb_cache(strip_history=True) for i, id in enumerate(case_ids): self.assertFalse(nohistory_cache.in_cache(id)) case = nohistory_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) == 0) more_case_ids = _make_some_cases(3) history_cache.populate(more_case_ids) nohistory_cache.populate(more_case_ids) for i, id in enumerate(more_case_ids): self.assertTrue(history_cache.in_cache(id)) case = history_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) > 0) for i, id in enumerate(more_case_ids): self.assertTrue(nohistory_cache.in_cache(id)) case = nohistory_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) == 0) def test_nowrap(self): case_ids = _make_some_cases(1) cache = self.interface.casedb_cache(wrap=False) case = cache.get(case_ids[0]) self.assertTrue(isinstance(case, dict)) self.assertFalse(isinstance(case, CommCareCase))
class SubmissionPost(object): def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, "'domain' is required" assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db if case_db: assert case_db.domain == domain self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 self.track_load = form_load_counter("form_submission", domain) def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form xforms[0] = _transform_instance_to_error(self.interface, error, instance) # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if any_migrations_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503) if not self.auth_context.is_valid(): return HttpResponseForbidden('Bad auth') if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message) def _post_process_form(self, xform): self._set_submission_properties(xform) found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def _get_success_message(self, instance, cases=None): ''' Formplayer requests get a detailed success message pointing to the form/case affected. All other requests get a generic message. Message is formatted with markdown. ''' if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID: return ' √ ' messages = [] user = CouchUser.get_by_user_id(instance.user_id) if not user or not user.is_web_user(): return _('Form successfully saved!') from corehq.apps.export.views.list import CaseExportListView, FormExportListView from corehq.apps.export.views.utils import can_view_case_exports, can_view_form_exports from corehq.apps.reports.views import CaseDataView, FormDataView form_link = case_link = form_export_link = case_export_link = None form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory' if has_permission_to_view_report(user, instance.domain, form_view): form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id]) case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport' if cases and has_permission_to_view_report(user, instance.domain, case_view): if len(cases) == 1: case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id]) else: case_link = ", ".join(["[{}]({})".format( c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id]) ) for c in cases]) if can_view_form_exports(user, instance.domain): form_export_link = reverse(FormExportListView.urlname, args=[instance.domain]) if cases and can_view_case_exports(user, instance.domain): case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain]) # Start with generic message messages.append(_('Form successfully saved!')) # Add link to form/case if possible if form_link and case_link: if len(cases) == 1: messages.append( _("You submitted [this form]({}), which affected [this case]({}).") .format(form_link, case_link)) else: messages.append( _("You submitted [this form]({}), which affected these cases: {}.") .format(form_link, case_link)) elif form_link: messages.append(_("You submitted [this form]({}).").format(form_link)) elif case_link: if len(cases) == 1: messages.append(_("Your form affected [this case]({}).").format(case_link)) else: messages.append(_("Your form affected these cases: {}.").format(case_link)) # Add link to all form/case exports if form_export_link and case_export_link: messages.append( _("Click to export your [case]({}) or [form]({}) data.") .format(case_export_link, form_export_link)) elif form_export_link: messages.append(_("Click to export your [form data]({}).").format(form_export_link)) elif case_export_link: messages.append(_("Click to export your [case data]({}).").format(case_export_link)) return "\n\n".join(messages) def run(self): self.track_load() failure_response = self._handle_basic_failure_modes() if failure_response: return FormProcessingResult(failure_response, None, [], [], 'known_failures') result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json()) submitted_form = result.submitted_form self._post_process_form(submitted_form) self._invalidate_caches(submitted_form) if submitted_form.is_submission_error_log: self.formdb.save_new_form(submitted_form) response = self.get_exception_response_and_log(submitted_form, self.path) return FormProcessingResult(response, None, [], [], 'submission_error_log') if submitted_form.xmlns == DEVICE_LOG_XMLNS: return self.process_device_log(submitted_form) cases = [] ledgers = [] submission_type = 'unknown' openrosa_kwargs = {} with result.get_locked_forms() as xforms: if len(xforms) > 1: self.track_load(len(xforms) - 1) if self.case_db: case_db_cache = self.case_db case_db_cache.cached_xforms.extend(xforms) else: case_db_cache = self.interface.casedb_cache( domain=self.domain, lock=True, deleted_ok=True, xforms=xforms, load_src="form_submission", ) with case_db_cache as case_db: instance = xforms[0] if instance.is_duplicate: with tracer.trace('submission.process_duplicate'): submission_type = 'duplicate' existing_form = xforms[1] stub = UnfinishedSubmissionStub.objects.filter( domain=instance.domain, xform_id=existing_form.form_id ).first() result = None if stub: from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False) elif existing_form.is_error: from corehq.form_processor.reprocess import reprocess_form result = reprocess_form(existing_form, lock_form=False) if result and result.error: submission_type = 'error' openrosa_kwargs['error_message'] = result.error if existing_form.is_error: openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE else: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE else: self.interface.save_processed_models([instance]) elif not instance.is_error: submission_type = 'normal' try: case_stock_result = self.process_xforms_for_cases(xforms, case_db) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: self._handle_known_error(e, instance, xforms) submission_type = 'error' openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms, case_stock_result) if openrosa_kwargs['error_message']: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE cases = case_stock_result.case_models ledgers = case_stock_result.stock_result.models_to_save openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases) elif instance.is_error: submission_type = 'error' response = self._get_open_rosa_response(instance, **openrosa_kwargs) return FormProcessingResult(response, instance, cases, ledgers, submission_type) def _conditionally_send_device_logs_to_sumologic(self, instance): url = getattr(settings, 'SUMOLOGIC_URL', None) if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER): SumoLogicLog(self.domain, instance).send_data(url) def _invalidate_caches(self, xform): for device_id in {None, xform.metadata.deviceID if xform.metadata else None}: self._invalidate_restore_payload_path_cache(xform, device_id) if ASYNC_RESTORE.enabled(self.domain): self._invalidate_async_restore_task_id_cache(xform, device_id) def _invalidate_restore_payload_path_cache(self, xform, device_id): """invalidate cached initial restores""" restore_payload_path_cache = RestorePayloadPathCache( domain=self.domain, user_id=xform.user_id, sync_log_id=xform.last_sync_token, device_id=device_id, ) restore_payload_path_cache.invalidate() def _invalidate_async_restore_task_id_cache(self, xform, device_id): async_restore_task_id_cache = AsyncRestoreTaskIdCache( domain=self.domain, user_id=xform.user_id, sync_log_id=self.last_sync_token, device_id=device_id, ) task_id = async_restore_task_id_cache.get_value() if task_id is not None: revoke_celery_task(task_id) async_restore_task_id_cache.invalidate() @tracer.wrap(name='submission.save_models') def save_processed_models(self, case_db, xforms, case_stock_result): instance = xforms[0] try: with unfinished_submission(instance) as unfinished_submission_stub: try: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result ) except PostSaveError: # mark the stub as saved if there's a post save error # but re-raise the error so that the re-processing queue picks it up unfinished_submission_stub.submission_saved() raise else: unfinished_submission_stub.submission_saved() self.do_post_save_actions(case_db, xforms, case_stock_result) except PostSaveError: return "Error performing post save operations" @staticmethod @tracer.wrap(name='submission.post_save_actions') def do_post_save_actions(case_db, xforms, case_stock_result): instance = xforms[0] try: case_stock_result.case_result.commit_dirtiness_flags() case_stock_result.stock_result.finalize() SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models) case_stock_result.case_result.close_extensions( case_db, "SubmissionPost-%s-close_extensions" % instance.form_id ) except PostSaveError: raise except Exception: notify_exception(get_request(), "Error performing post save actions during form processing", { 'domain': instance.domain, 'form_id': instance.form_id, }) raise PostSaveError @staticmethod @tracer.wrap(name='submission.process_cases_and_stock') def process_xforms_for_cases(xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on cases = case_db.get_cases_for_saving(modified_on_date) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): return self.run().response @staticmethod def _fire_post_save_signals(instance, cases): from casexml.apps.case.signals import case_post_save error_message = "Error occurred during form submission post save (%s)" error_details = {'domain': instance.domain, 'form_id': instance.form_id} results = successful_form_received.send_robust(None, xform=instance) has_errors = log_signal_errors(results, error_message, error_details) for case in cases: results = case_post_save.send_robust(case.__class__, case=case) has_errors |= log_signal_errors(results, error_message, error_details) if has_errors: raise PostSaveError def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None): if self.is_openrosa_version3: instance_ok = instance.is_normal or instance.is_duplicate has_error = error_message or error_nature if instance_ok and not has_error: response = openrosa_response.get_openarosa_success_response(message=success_message) else: error_message = error_message or instance.problem response = self.get_retry_response(error_message, error_nature) else: if instance.is_normal: response = openrosa_response.get_openarosa_success_response() else: response = self.get_v2_submit_error_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_v2_submit_error_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_retry_response(message, nature): """Returns a 422(Unprocessable Entity) response, mobile will retry this submission """ return OpenRosaResponse( message=message, nature=nature, status=422, ).response() @staticmethod def get_exception_response_and_log(error_instance, path): logging.exception( "Problem receiving submission to %s. Doc id: %s, Error %s" % ( path, error_instance.form_id, error_instance.problem ) ) return OpenRosaResponse( message="There was an error processing the form: %s" % error_instance.problem, nature=ResponseNature.SUBMIT_ERROR, status=500, ).response() @tracer.wrap(name='submission.process_device_log') def process_device_log(self, device_log_form): self._conditionally_send_device_logs_to_sumologic(device_log_form) ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.NO_DEVICE_LOG_ENVS if not ignore_device_logs: try: process_device_log(self.domain, device_log_form) except Exception as e: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) e.sentry_capture = False raise response = self._get_open_rosa_response(device_log_form) return FormProcessingResult(response, device_log_form, [], [], 'device-log')
class SubmissionPost(object): def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.auth_context = self.auth_context.to_json() xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form xforms[0] = _transform_instance_to_error(self.interface, error, instance) # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if any_migrations_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503) if not self.auth_context.is_valid(): return HttpResponseForbidden('Bad auth') if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message) def _post_process_form(self, xform): self._set_submission_properties(xform) found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def run(self): failure_response = self._handle_basic_failure_modes() if failure_response: return FormProcessingResult(failure_response, None, [], [], 'known_failures') result = process_xform_xml(self.domain, self.instance, self.attachments) submitted_form = result.submitted_form self._post_process_form(submitted_form) self._invalidate_caches(submitted_form) submission_type = None if submitted_form.is_submission_error_log: self.formdb.save_new_form(submitted_form) response = self.get_exception_response_and_log(submitted_form, self.path) return FormProcessingResult(response, None, [], [], 'submission_error_log') cases = [] ledgers = [] submission_type = 'unknown' response_nature = error_message = None with result.get_locked_forms() as xforms: from casexml.apps.case.xform import get_and_check_xform_domain domain = get_and_check_xform_domain(xforms[0]) if self.case_db: assert self.case_db.domain == domain case_db_cache = self.case_db case_db_cache.cached_xforms.extend(xforms) else: case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) with case_db_cache as case_db: instance = xforms[0] if instance.xmlns == DEVICE_LOG_XMLNS: submission_type = 'device_log' try: process_device_log(self.domain, instance) except Exception: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) raise elif instance.is_duplicate: submission_type = 'duplicate' existing_form = xforms[1] stub = UnfinishedSubmissionStub.objects.filter( domain=instance.domain, xform_id=existing_form.form_id ).first() result = None if stub: from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False) elif existing_form.is_error: from corehq.form_processor.reprocess import reprocess_form result = reprocess_form(existing_form, lock_form=False) if result and result.error: submission_type = 'error' error_message = result.error if existing_form.is_error: response_nature = ResponseNature.PROCESSING_FAILURE else: response_nature = ResponseNature.POST_PROCESSING_FAILIRE else: self.interface.save_processed_models([instance]) elif not instance.is_error: submission_type = 'normal' try: case_stock_result = self.process_xforms_for_cases(xforms, case_db) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: self._handle_known_error(e, instance, xforms) submission_type = 'error' response_nature = ResponseNature.PROCESSING_FAILURE except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True error_message = self.save_processed_models(case_db, xforms, case_stock_result) if error_message: response_nature = ResponseNature.POST_PROCESSING_FAILIRE cases = case_stock_result.case_models ledgers = case_stock_result.stock_result.models_to_save elif instance.is_error: submission_type = 'error' response = self._get_open_rosa_response(instance, error_message, response_nature) return FormProcessingResult(response, instance, cases, ledgers, submission_type) def _invalidate_caches(self, xform): for device_id in {None, xform.metadata.deviceID if xform.metadata else None}: self._invalidate_restore_payload_path_cache(xform, device_id) if ASYNC_RESTORE.enabled(self.domain): self._invalidate_async_restore_task_id_cache(xform, device_id) def _invalidate_restore_payload_path_cache(self, xform, device_id): """invalidate cached initial restores""" restore_payload_path_cache = RestorePayloadPathCache( domain=self.domain, user_id=xform.user_id, sync_log_id=xform.last_sync_token, device_id=device_id, ) restore_payload_path_cache.invalidate() def _invalidate_async_restore_task_id_cache(self, xform, device_id): async_restore_task_id_cache = AsyncRestoreTaskIdCache( domain=self.domain, user_id=xform.user_id, sync_log_id=self.last_sync_token, device_id=device_id, ) task_id = async_restore_task_id_cache.get_value() if task_id is not None: revoke_celery_task(task_id) async_restore_task_id_cache.invalidate() def save_processed_models(self, case_db, xforms, case_stock_result): instance = xforms[0] try: with unfinished_submission(instance) as unfinished_submission_stub: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result ) if unfinished_submission_stub: unfinished_submission_stub.saved = True unfinished_submission_stub.save() self.do_post_save_actions(case_db, xforms, case_stock_result) except PostSaveError: return "Error performing post save operations" @staticmethod def do_post_save_actions(case_db, xforms, case_stock_result): instance = xforms[0] try: case_stock_result.case_result.commit_dirtiness_flags() case_stock_result.stock_result.finalize() SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models) case_stock_result.case_result.close_extensions( case_db, "SubmissionPost-%s-close_extensions" % instance.form_id ) except PostSaveError: raise except Exception: notify_exception(get_request(), "Error performing post save actions during form processing", { 'domain': instance.domain, 'form_id': instance.form_id, }) raise PostSaveError @staticmethod def process_xforms_for_cases(xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on cases = case_db.get_cases_for_saving(modified_on_date) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): return self.run().response @staticmethod def _fire_post_save_signals(instance, cases): from casexml.apps.case.signals import case_post_save error_message = "Error occurred during form submission post save (%s)" error_details = {'domain': instance.domain, 'form_id': instance.form_id} results = successful_form_received.send_robust(None, xform=instance) has_errors = log_signal_errors(results, error_message, error_details) for case in cases: results = case_post_save.send_robust(case.__class__, case=case) has_errors |= log_signal_errors(results, error_message, error_details) if has_errors: raise PostSaveError def _get_open_rosa_response(self, instance, error_message=None, error_nature=None): if self.is_openrosa_version3: instance_ok = instance.is_normal or instance.is_duplicate has_error = error_message or error_nature if instance_ok and not has_error: response = openrosa_response.SUCCESS_RESPONSE else: error_message = error_message or instance.problem response = self.get_retry_response(error_message, error_nature) else: if instance.is_normal: response = openrosa_response.SUCCESS_RESPONSE else: response = self.get_v2_submit_error_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_v2_submit_error_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_retry_response(message, nature): """Returns a 422(Unprocessable Entity) response, mobile will retry this submission """ return OpenRosaResponse( message=message, nature=nature, status=422, ).response() @staticmethod def get_exception_response_and_log(error_instance, path): logging.exception( u"Problem receiving submission to %s. Doc id: %s, Error %s" % ( path, error_instance.form_id, error_instance.problem ) ) return OpenRosaResponse( message="There was an error processing the form: %s" % error_instance.problem, nature=ResponseNature.SUBMIT_ERROR, status=500, ).response()
class SubmissionPost(object): failed_auth_response = HttpResponseForbidden('Bad auth') def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.auth_context = self.auth_context.to_json() xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form instance = _transform_instance_to_error(self.interface, error, instance) xforms[0] = instance # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if timezone_migration_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503), None, [] if not self.auth_context.is_valid(): return self.failed_auth_response, None, [] if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message), None, [] def _post_process_form(self, xform): self._set_submission_properties(xform) if xform.is_submission_error_log: found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def run(self): failure_result = self._handle_basic_failure_modes() if failure_result: return failure_result result = process_xform_xml(self.domain, self.instance, self.attachments) submitted_form = result.submitted_form self._post_process_form(submitted_form) if submitted_form.is_submission_error_log: self.formdb.save_new_form(submitted_form) response = self.get_exception_response_and_log(submitted_form, self.path) return response, None, [] cases = [] with result.get_locked_forms() as xforms: instance = xforms[0] if instance.xmlns == DEVICE_LOG_XMLNS: try: process_device_log(self.domain, instance) except Exception: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) raise elif instance.is_duplicate: self.interface.save_processed_models([instance]) elif not instance.is_error: try: case_stock_result = self.process_xforms_for_cases(xforms) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError) as e: self._handle_known_error(e, instance, xforms) except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True self.save_processed_models(xforms, case_stock_result) cases = case_stock_result.case_models errors = self.process_signals(instance) response = self._get_open_rosa_response(instance, errors) return response, instance, cases def save_processed_models(self, xforms, case_stock_result): from casexml.apps.case.signals import case_post_save instance = xforms[0] with unfinished_submission(instance) as unfinished_submission_stub: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result ) unfinished_submission_stub.saved = True unfinished_submission_stub.save() case_stock_result.case_result.commit_dirtiness_flags() case_stock_result.stock_result.finalize() for case in case_stock_result.case_models: case_post_save.send(case.__class__, case=case) case_stock_result.case_result.close_extensions() def process_xforms_for_cases(self, xforms): from casexml.apps.case.xform import get_and_check_xform_domain from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] domain = get_and_check_xform_domain(instance) with self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db: case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(instance.received_on) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): response, _, _ = self.run() return response def process_signals(self, instance): feedback = successful_form_received.send_robust(None, xform=instance) errors = [] for func, resp in feedback: if resp and isinstance(resp, Exception): error_message = unicode(resp) logging.error(( u"Receiver app: problem sending " u"post-save signal %s for xform %s: %s: %s" ) % (func, instance.form_id, type(resp).__name__, error_message)) errors.append(error_message) if errors: self.interface.xformerror_from_xform_instance(instance, ", ".join(errors)) self.formdb.update_form_problem_and_state(instance) return errors @staticmethod def get_failed_auth_response(): return HttpResponseForbidden('Bad auth') def _get_open_rosa_response(self, instance, errors): if instance.is_normal: response = self.get_success_response(instance, errors) else: response = self.get_failure_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_success_response(doc, errors): if errors: response = OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() else: response = OpenRosaResponse( # would have done ✓ but our test Nokias' fonts don't have that character message=u' √ ', nature=ResponseNature.SUBMIT_SUCCESS, status=201, ).response() return response @staticmethod def get_failure_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_exception_response_and_log(error_instance, path): logging.exception( u"Problem receiving submission to %s. Doc id: %s, Error %s" % ( path, error_instance.form_id, error_instance.problem ) ) return OpenRosaResponse( message=("The sever got itself into big trouble! " "Details: %s" % error_instance.problem), nature=ResponseNature.SUBMIT_ERROR, status=500, ).response()