class CaseDbCacheCouchOnlyTest(TestCase):
def setUp(self):
super(CaseDbCacheCouchOnlyTest, self).setUp()
self.interface = FormProcessorInterface()
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
cache = CaseDbCacheCouch()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
def testStripHistory(self):
case_ids = _make_some_cases(3)
history_cache = self.interface.casedb_cache()
for i, id in enumerate(case_ids):
self.assertFalse(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
nohistory_cache = self.interface.casedb_cache(strip_history=True)
for i, id in enumerate(case_ids):
self.assertFalse(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
more_case_ids = _make_some_cases(3)
history_cache.populate(more_case_ids)
nohistory_cache.populate(more_case_ids)
for i, id in enumerate(more_case_ids):
self.assertTrue(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
for i, id in enumerate(more_case_ids):
self.assertTrue(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
def test_nowrap(self):
case_ids = _make_some_cases(1)
cache = self.interface.casedb_cache(wrap=False)
case = cache.get(case_ids[0])
self.assertTrue(isinstance(case, dict))
self.assertFalse(isinstance(case, CommCareCase))
class CaseDbCacheTest(TestCase):
"""
Tests the functionality of the CaseDbCache object
"""
def setUp(self):
super(CaseDbCacheTest, self).setUp()
self.interface = FormProcessorInterface()
@run_with_all_backends
def testDomainCheck(self):
id = uuid.uuid4().hex
post_case_blocks(
[CaseBlock(create=True, case_id=id, user_id='some-user').as_xml()],
{'domain': 'good-domain'})
bad_cache = self.interface.casedb_cache(domain='bad-domain')
try:
bad_cache.get(id)
self.fail('domain security check failed to raise exception')
except IllegalCaseId:
pass
good_cache = self.interface.casedb_cache(domain='good-domain')
case = good_cache.get(id)
self.assertEqual(
'some-user',
case.user_id) # just sanity check it's the right thing
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
cache = CaseDbCacheCouch()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
@run_with_all_backends
def testGetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i),
case.dynamic_case_properties()['my_index'])
for id in case_ids:
self.assertTrue(cache.in_cache(id))
def _perfom_post_save_actions(form, save=True):
interface = FormProcessorInterface(form.domain)
cache = interface.casedb_cache(
domain=form.domain, lock=False, deleted_ok=True, xforms=[form],
load_src="reprocess_form_post_save",
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
case_models = case_stock_result.case_models
if interface.use_sql_domain:
forms = ProcessedForms(form, None)
stock_result = case_stock_result.stock_result
try:
FormProcessorSQL.publish_changes_to_kafka(forms, case_models, stock_result)
except Exception:
error_message = "Error publishing to kafka"
return ReprocessingResult(form, None, None, error_message)
try:
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
except PostSaveError:
error_message = "Error performing post save operations"
return ReprocessingResult(form, None, None, error_message)
return ReprocessingResult(form, case_models, None, None)
class CaseDbCacheCouchOnlyTest(TestCase):
def setUp(self):
super(CaseDbCacheCouchOnlyTest, self).setUp()
self.interface = FormProcessorInterface()
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
cache = CaseDbCacheCouch()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
def test_nowrap(self):
case_ids = _make_some_cases(1)
cache = self.interface.casedb_cache(wrap=False)
case = cache.get(case_ids[0])
self.assertTrue(isinstance(case, dict))
self.assertFalse(isinstance(case, CommCareCase))
class CaseDbCacheCouchOnlyTest(TestCase):
def setUp(self):
super(CaseDbCacheCouchOnlyTest, self).setUp()
self.interface = FormProcessorInterface()
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
cache = CaseDbCacheCouch()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
def test_nowrap(self):
case_ids = _make_some_cases(1)
cache = self.interface.casedb_cache(wrap=False)
case = cache.get(case_ids[0])
self.assertTrue(isinstance(case, dict))
self.assertFalse(isinstance(case, CommCareCase))
def _perfom_post_save_actions(form, save=True):
interface = FormProcessorInterface(form.domain)
cache = interface.casedb_cache(
domain=form.domain, lock=False, deleted_ok=True, xforms=[form]
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
try:
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
except PostSaveError:
error_message = "Error performing post save operations"
return ReprocessingResult(form, None, None, error_message)
return ReprocessingResult(form, case_stock_result.case_models, None, None)
class CaseDbCacheTest(TestCase):
"""
Tests the functionality of the CaseDbCache object
"""
def setUp(self):
super(CaseDbCacheTest, self).setUp()
self.interface = FormProcessorInterface()
def testDomainCheck(self):
id = uuid.uuid4().hex
post_case_blocks([
CaseBlock.deprecated_init(
create=True, case_id=id, user_id='some-user').as_xml()
], {'domain': 'good-domain'})
bad_cache = self.interface.casedb_cache(domain='bad-domain')
try:
bad_cache.get(id)
self.fail('domain security check failed to raise exception')
except IllegalCaseId:
pass
good_cache = self.interface.casedb_cache(domain='good-domain')
case = good_cache.get(id)
self.assertEqual(
'some-user',
case.user_id) # just sanity check it's the right thing
def testGetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i),
case.dynamic_case_properties()['my_index'])
for id in case_ids:
self.assertTrue(cache.in_cache(id))
def _get_case_and_ledger_updates(domain, sql_form):
"""
Get a CaseStockProcessingResult with the appropriate cases and ledgers to
be saved.
See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent
section of the form-processing code.
"""
from corehq.apps.commtrack.processing import process_stock
interface = FormProcessorInterface(domain)
assert sql_form.domain
xforms = [sql_form]
with interface.casedb_cache(
domain=domain,
lock=False,
deleted_ok=True,
xforms=xforms,
load_src="couchsqlmigration",
) as case_db:
touched_cases = FormProcessorInterface(domain).get_cases_from_forms(
case_db, xforms)
extensions_to_close = get_all_extensions_to_close(
domain, list(touched_cases.values()))
case_result = CaseProcessingResult(
domain,
[update.case for update in touched_cases.values()],
[], # ignore dirtiness_flags,
extensions_to_close)
for case in case_result.cases:
case_db.post_process_case(case, sql_form)
case_db.mark_changed(case)
cases = case_result.cases
try:
stock_result = process_stock(xforms, case_db)
cases = case_db.get_cases_for_saving(sql_form.received_on)
stock_result.populate_models()
except MissingFormXml:
stock_result = None
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def _get_case_and_ledger_updates(domain, sql_form):
"""
Get a CaseStockProcessingResult with the appropriate cases and ledgers to
be saved.
See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent
section of the form-processing code.
"""
from casexml.apps.case.xform import get_and_check_xform_domain
from corehq.apps.commtrack.processing import process_stock
interface = FormProcessorInterface(domain)
get_and_check_xform_domain(sql_form)
xforms = [sql_form]
# todo: I think this can be changed to lock=False
with interface.casedb_cache(domain=domain,
lock=True,
deleted_ok=True,
xforms=xforms) as case_db:
touched_cases = FormProcessorInterface(domain).get_cases_from_forms(
case_db, xforms)
extensions_to_close = get_all_extensions_to_close(
domain, touched_cases.values())
case_result = CaseProcessingResult(
domain,
[update.case for update in touched_cases.values()],
[], # ignore dirtiness_flags,
extensions_to_close)
# todo: is this necessary?
for case in case_result.cases:
case_db.mark_changed(case)
stock_result = process_stock(xforms, case_db)
cases = case_db.get_cases_for_saving(sql_form.received_on)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def _get_case_and_ledger_updates(domain, sql_form):
"""
Get a CaseStockProcessingResult with the appropriate cases and ledgers to
be saved.
See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent
section of the form-processing code.
"""
from corehq.apps.commtrack.processing import process_stock
interface = FormProcessorInterface(domain)
assert sql_form.domain
xforms = [sql_form]
with interface.casedb_cache(
domain=domain, lock=False, deleted_ok=True, xforms=xforms,
load_src="couchsqlmigration",
) as case_db:
touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms)
extensions_to_close = get_all_extensions_to_close(domain, list(touched_cases.values()))
case_result = CaseProcessingResult(
domain,
[update.case for update in touched_cases.values()],
[], # ignore dirtiness_flags,
extensions_to_close
)
for case in case_result.cases:
case_db.post_process_case(case, sql_form)
case_db.mark_changed(case)
stock_result = process_stock(xforms, case_db)
cases = case_db.get_cases_for_saving(sql_form.received_on)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
class SubmissionPost(object):
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.auth_context = self.auth_context.to_json()
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
instance = _transform_instance_to_error(self.interface, error, instance)
xforms[0] = instance
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message)
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def run(self):
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form.user_id)
submission_type = None
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
cases = []
ledgers = []
submission_type = 'unknown'
with result.get_locked_forms() as xforms:
from casexml.apps.case.xform import get_and_check_xform_domain
domain = get_and_check_xform_domain(xforms[0])
if self.case_db:
assert self.case_db.domain == domain
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms)
with case_db_cache as case_db:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
submission_type = 'device_log'
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
submission_type = 'duplicate'
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
self.save_processed_models(xforms, case_stock_result)
case_stock_result.case_result.close_extensions(case_db)
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
elif instance.is_error:
submission_type = 'error'
errors = self.process_signals(instance)
response = self._get_open_rosa_response(instance, errors)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
@property
def _cache(self):
return get_redis_default_cache()
@property
def _restore_cache_key(self):
from casexml.apps.phone.restore import restore_cache_key
return restore_cache_key
def _invalidate_caches(self, user_id):
"""invalidate cached initial restores"""
initial_restore_cache_key = self._restore_cache_key(
self.domain,
RESTORE_CACHE_KEY_PREFIX,
user_id,
version=V2
)
self._cache.delete(initial_restore_cache_key)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_caches(user_id)
def _invalidate_async_caches(self, user_id):
cache_key = self._restore_cache_key(self.domain, ASYNC_RESTORE_CACHE_KEY_PREFIX, user_id)
task_id = self._cache.get(cache_key)
if task_id is not None:
revoke_celery_task(task_id)
self._cache.delete(cache_key)
def save_processed_models(self, xforms, case_stock_result):
from casexml.apps.case.signals import case_post_save
instance = xforms[0]
with unfinished_submission(instance) as unfinished_submission_stub:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
for case in case_stock_result.case_models:
case_post_save.send(case.__class__, case=case)
@staticmethod
def process_xforms_for_cases(xforms, case_db):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
modified_on_date = instance.received_on
if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on:
modified_on_date = instance.edited_on
cases = case_db.get_cases_for_saving(modified_on_date)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
def process_signals(self, instance):
feedback = successful_form_received.send_robust(None, xform=instance)
errors = []
for func, resp in feedback:
if resp and isinstance(resp, Exception):
error_message = unicode(resp)
logging.error((
u"Receiver app: problem sending "
u"post-save signal %s for xform %s: %s: %s"
) % (func, instance.form_id, type(resp).__name__, error_message))
errors.append(error_message)
if errors:
self.interface.xformerror_from_xform_instance(instance, ", ".join(errors))
self.formdb.update_form_problem_and_state(instance)
return errors
def _get_open_rosa_response(self, instance, errors):
if instance.is_normal and not errors:
response = self.get_success_response()
else:
response = self.get_failure_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_success_response():
return OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u' √ ',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
@staticmethod
def submission_ignored_response():
return OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u'√ (this submission was ignored)',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
@staticmethod
def get_failure_response(doc):
return OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
u"Problem receiving submission to %s. Doc id: %s, Error %s" % (
path,
error_instance.form_id,
error_instance.problem
)
)
return OpenRosaResponse(
message="There was an error processing the form: %s" % error_instance.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
@staticmethod
def get_blacklisted_response():
return OpenRosaResponse(
message=("This submission was blocked because of an unusual volume "
"of submissions from this project space. Please contact "
"support to resolve."),
nature=ResponseNature.SUBMIT_ERROR,
status=509,
).response()
def reprocess_form(form, save=True, lock_form=True):
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(
form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases(
[form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
error_message = '{}: {}'.format(
type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(
form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
else:
class CaseDbCacheTest(TestCase):
"""
Tests the functionality of the CaseDbCache object
"""
def setUp(self):
super(CaseDbCacheTest, self).setUp()
self.interface = FormProcessorInterface()
def testDomainCheck(self):
id = uuid.uuid4().hex
post_case_blocks([
CaseBlock(
create=True, case_id=id,
user_id='some-user'
).as_xml()
], {'domain': 'good-domain'}
)
bad_cache = self.interface.casedb_cache(domain='bad-domain')
try:
bad_cache.get(id)
self.fail('domain security check failed to raise exception')
except IllegalCaseId:
pass
good_cache = self.interface.casedb_cache(domain='good-domain')
case = good_cache.get(id)
self.assertEqual('some-user', case.user_id) # just sanity check it's the right thing
def testGetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i), case.dynamic_case_properties()['my_index'])
for id in case_ids:
self.assertTrue(cache.in_cache(id))
def testSetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for id in case_ids:
cache.set(id, CaseAccessors().get_case(id))
for i, id in enumerate(case_ids):
self.assertTrue(cache.in_cache(id))
case = cache.get(id)
self.assertEqual(str(i), case.dynamic_case_properties()['my_index'])
def testPopulate(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
cache.populate(case_ids)
for id in case_ids:
self.assertTrue(cache.in_cache(id))
# sanity check
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i), case.dynamic_case_properties()['my_index'])
class CaseDbCacheCouchOnlyTest(TestCase):
def setUp(self):
super(CaseDbCacheCouchOnlyTest, self).setUp()
self.interface = FormProcessorInterface()
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
cache = CaseDbCacheCouch()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
def testStripHistory(self):
case_ids = _make_some_cases(3)
history_cache = self.interface.casedb_cache()
for i, id in enumerate(case_ids):
self.assertFalse(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
nohistory_cache = self.interface.casedb_cache(strip_history=True)
for i, id in enumerate(case_ids):
self.assertFalse(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
more_case_ids = _make_some_cases(3)
history_cache.populate(more_case_ids)
nohistory_cache.populate(more_case_ids)
for i, id in enumerate(more_case_ids):
self.assertTrue(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
for i, id in enumerate(more_case_ids):
self.assertTrue(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
def test_nowrap(self):
case_ids = _make_some_cases(1)
cache = self.interface.casedb_cache(wrap=False)
case = cache.get(case_ids[0])
self.assertTrue(isinstance(case, dict))
self.assertFalse(isinstance(case, CommCareCase))
class SubmissionPost(object):
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
xforms[0] = _transform_instance_to_error(self.interface, error, instance)
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message)
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def _get_success_message(self, instance, cases=None):
'''
Formplayer requests get a detailed success message pointing to the form/case affected.
All other requests get a generic message.
Message is formatted with markdown.
'''
if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID:
return ' √ '
messages = []
user = CouchUser.get_by_user_id(instance.user_id)
if not user or not user.is_web_user():
return _('Form successfully saved!')
from corehq.apps.export.views import CaseExportListView, FormExportListView
from corehq.apps.reports.views import CaseDataView, FormDataView
form_link = case_link = form_export_link = case_export_link = None
form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory'
if has_permission_to_view_report(user, instance.domain, form_view):
form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id])
case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport'
if cases and has_permission_to_view_report(user, instance.domain, case_view):
if len(cases) == 1:
case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id])
else:
case_link = ", ".join(["[{}]({})".format(
c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id])
) for c in cases])
if can_view_form_exports(user, instance.domain):
form_export_link = reverse(FormExportListView.urlname, args=[instance.domain])
if cases and can_view_case_exports(user, instance.domain):
case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain])
# Start with generic message
messages.append(_('Form successfully saved!'))
# Add link to form/case if possible
if form_link and case_link:
if len(cases) == 1:
messages.append(
_("You submitted [this form]({}), which affected [this case]({}).")
.format(form_link, case_link))
else:
messages.append(
_("You submitted [this form]({}), which affected these cases: {}.")
.format(form_link, case_link))
elif form_link:
messages.append(_("You submitted [this form]({}).").format(form_link))
elif case_link:
if len(cases) == 1:
messages.append(_("Your form affected [this case]({}).").format(case_link))
else:
messages.append(_("Your form affected these cases: {}.").format(case_link))
# Add link to all form/case exports
if form_export_link and case_export_link:
messages.append(
_("Click to export your [case]({}) or [form]({}) data.")
.format(case_export_link, form_export_link))
elif form_export_link:
messages.append(_("Click to export your [form data]({}).").format(form_export_link))
elif case_export_link:
messages.append(_("Click to export your [case data]({}).").format(case_export_link))
return "\n\n".join(messages)
def run(self):
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json())
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
submission_type = None
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
cases = []
ledgers = []
submission_type = 'unknown'
openrosa_kwargs = {}
with result.get_locked_forms() as xforms:
from casexml.apps.case.xform import get_and_check_xform_domain
domain = get_and_check_xform_domain(xforms[0])
if self.case_db:
assert self.case_db.domain == domain
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms)
with case_db_cache as case_db:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
submission_type = 'device_log'
self._conditionally_send_device_logs_to_sumologic(instance)
# ignore temporarily till we migrate DeviceReportEntry id to bigint
ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.ICDS_ENVS
if not ignore_device_logs and instance.xmlns == DEVICE_LOG_XMLNS:
submission_type = 'device_log'
try:
process_device_log(self.domain, instance)
except Exception as e:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
e.sentry_capture = False
raise
elif instance.is_duplicate:
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
openrosa_kwargs['error_message'] = result.error
if existing_form.is_error:
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
else:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms,
case_stock_result)
if openrosa_kwargs['error_message']:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases)
elif instance.is_error:
submission_type = 'error'
response = self._get_open_rosa_response(instance, **openrosa_kwargs)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
def _conditionally_send_device_logs_to_sumologic(self, instance):
url = getattr(settings, 'SUMOLOGIC_URL', None)
if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER):
send_device_logs_to_sumologic.delay(self.domain, instance, url)
def _invalidate_caches(self, xform):
for device_id in {None, xform.metadata.deviceID if xform.metadata else None}:
self._invalidate_restore_payload_path_cache(xform, device_id)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_restore_task_id_cache(xform, device_id)
def _invalidate_restore_payload_path_cache(self, xform, device_id):
"""invalidate cached initial restores"""
restore_payload_path_cache = RestorePayloadPathCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=xform.last_sync_token,
device_id=device_id,
)
restore_payload_path_cache.invalidate()
def _invalidate_async_restore_task_id_cache(self, xform, device_id):
async_restore_task_id_cache = AsyncRestoreTaskIdCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=self.last_sync_token,
device_id=device_id,
)
task_id = async_restore_task_id_cache.get_value()
if task_id is not None:
revoke_celery_task(task_id)
async_restore_task_id_cache.invalidate()
def save_processed_models(self, case_db, xforms, case_stock_result):
instance = xforms[0]
try:
with unfinished_submission(instance) as unfinished_submission_stub:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
if unfinished_submission_stub:
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
self.do_post_save_actions(case_db, xforms, case_stock_result)
except PostSaveError:
return "Error performing post save operations"
@staticmethod
def do_post_save_actions(case_db, xforms, case_stock_result):
instance = xforms[0]
try:
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models)
case_stock_result.case_result.close_extensions(
case_db,
"SubmissionPost-%s-close_extensions" % instance.form_id
)
except PostSaveError:
raise
except Exception:
notify_exception(get_request(), "Error performing post save actions during form processing", {
'domain': instance.domain,
'form_id': instance.form_id,
})
raise PostSaveError
@staticmethod
def process_xforms_for_cases(xforms, case_db):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
modified_on_date = instance.received_on
if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on:
modified_on_date = instance.edited_on
cases = case_db.get_cases_for_saving(modified_on_date)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
@staticmethod
def _fire_post_save_signals(instance, cases):
from casexml.apps.case.signals import case_post_save
error_message = "Error occurred during form submission post save (%s)"
error_details = {'domain': instance.domain, 'form_id': instance.form_id}
results = successful_form_received.send_robust(None, xform=instance)
has_errors = log_signal_errors(results, error_message, error_details)
for case in cases:
results = case_post_save.send_robust(case.__class__, case=case)
has_errors |= log_signal_errors(results, error_message, error_details)
if has_errors:
raise PostSaveError
def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None):
if self.is_openrosa_version3:
instance_ok = instance.is_normal or instance.is_duplicate
has_error = error_message or error_nature
if instance_ok and not has_error:
response = openrosa_response.get_openarosa_success_response(message=success_message)
else:
error_message = error_message or instance.problem
response = self.get_retry_response(error_message, error_nature)
else:
if instance.is_normal:
response = openrosa_response.get_openarosa_success_response()
else:
response = self.get_v2_submit_error_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_v2_submit_error_response(doc):
return OpenRosaResponse(
message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201,
).response()
@staticmethod
def get_retry_response(message, nature):
"""Returns a 422(Unprocessable Entity) response, mobile will retry this submission
"""
return OpenRosaResponse(
message=message, nature=nature, status=422,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
"Problem receiving submission to %s. Doc id: %s, Error %s" % (
path,
error_instance.form_id,
error_instance.problem
)
)
return OpenRosaResponse(
message="There was an error processing the form: %s" % error_instance.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
def reprocess_form(form, save=True, lock_form=True):
if lock_form:
# track load if locking; otherise it will be tracked elsewhere
form_load_counter("reprocess_form", form.domain)()
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form],
load_src="reprocess_form",
)
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
error_message = '{}: {}'.format(type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id, detail, lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s', ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
else:
if save:
interface.processor.save_processed_models([form], cases, stock_result)
from casexml.apps.stock.models import StockTransaction
ledgers = [
model
for model in stock_result.models_to_save
if isinstance(model, StockTransaction)
]
for ledger in ledgers:
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
return ReprocessingResult(form, cases, ledgers, None)
class SubmissionPost(object):
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None, force_logs=False,
timing_context=None):
assert domain, "'domain' is required"
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
if case_db:
assert case_db.domain == domain
self.force_logs = force_logs
self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3
self.track_load = form_load_counter("form_submission", domain)
self.timing_context = timing_context or TimingContext()
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
xforms[0] = _transform_instance_to_error(self.interface, error, instance)
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def _get_success_message(self, instance, cases=None):
'''
Formplayer requests get a detailed success message pointing to the form/case affected.
All other requests get a generic message.
Message is formatted with markdown.
'''
if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID:
return ' √ '
messages = []
user = CouchUser.get_by_user_id(instance.user_id)
if not user or not user.is_web_user():
return _('Form successfully saved!')
from corehq.apps.export.views.list import CaseExportListView, FormExportListView
from corehq.apps.export.views.utils import can_view_case_exports, can_view_form_exports
from corehq.apps.reports.views import CaseDataView, FormDataView
form_link = case_link = form_export_link = case_export_link = None
form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory'
if has_permission_to_view_report(user, instance.domain, form_view):
form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id])
case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport'
if cases and has_permission_to_view_report(user, instance.domain, case_view):
if len(cases) == 1:
case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id])
else:
case_link = ", ".join(["[{}]({})".format(
c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id])
) for c in cases])
if can_view_form_exports(user, instance.domain):
form_export_link = reverse(FormExportListView.urlname, args=[instance.domain])
if cases and can_view_case_exports(user, instance.domain):
case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain])
# Start with generic message
messages.append(_('Form successfully saved!'))
# Add link to form/case if possible
if form_link and case_link:
if len(cases) == 1:
messages.append(
_("You submitted [this form]({}), which affected [this case]({}).")
.format(form_link, case_link))
else:
messages.append(
_("You submitted [this form]({}), which affected these cases: {}.")
.format(form_link, case_link))
elif form_link:
messages.append(_("You submitted [this form]({}).").format(form_link))
elif case_link:
if len(cases) == 1:
messages.append(_("Your form affected [this case]({}).").format(case_link))
else:
messages.append(_("Your form affected these cases: {}.").format(case_link))
# Add link to all form/case exports
if form_export_link and case_export_link:
messages.append(
_("Click to export your [case]({}) or [form]({}) data.")
.format(case_export_link, form_export_link))
elif form_export_link:
messages.append(_("Click to export your [form data]({}).").format(form_export_link))
elif case_export_link:
messages.append(_("Click to export your [case data]({}).").format(case_export_link))
return "\n\n".join(messages)
def run(self):
self.track_load()
with self.timing_context("process_xml"):
report_submission_usage(self.domain)
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json())
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
if submitted_form.is_submission_error_log:
logging.info('Processing form %s as a submission error', submitted_form.form_id)
XFormInstance.objects.save_new_form(submitted_form)
response = None
try:
xml = self.instance.decode()
except UnicodeDecodeError:
pass
else:
if 'log_subreport' in xml:
response = self.get_exception_response_and_log(
'Badly formed device log', submitted_form, self.path
)
if not response:
response = self.get_exception_response_and_log(
'Problem receiving submission', submitted_form, self.path
)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
if submitted_form.xmlns == SYSTEM_ACTION_XMLNS:
logging.info('Processing form %s as a system action', submitted_form.form_id)
with self.timing_context("process_system_action"):
return self.handle_system_action(submitted_form)
if submitted_form.xmlns == DEVICE_LOG_XMLNS:
logging.info('Processing form %s as a device log', submitted_form.form_id)
with self.timing_context("process_device_log"):
return self.process_device_log(submitted_form)
# Begin Normal Form Processing
self._log_form_details(submitted_form)
cases = []
ledgers = []
submission_type = 'unknown'
openrosa_kwargs = {}
with result.get_locked_forms() as xforms:
if len(xforms) > 1:
self.track_load(len(xforms) - 1)
if self.case_db:
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(
domain=self.domain, lock=True, deleted_ok=True,
xforms=xforms, load_src="form_submission",
)
with case_db_cache as case_db:
instance = xforms[0]
if instance.is_duplicate:
with self.timing_context("process_duplicate"), tracer.trace('submission.process_duplicate'):
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
openrosa_kwargs['error_message'] = result.error
if existing_form.is_error:
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
else:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db, self.timing_context)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms,
case_stock_result)
if openrosa_kwargs['error_message']:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
report_case_usage(self.domain, len(cases))
openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases)
elif instance.is_error:
submission_type = 'error'
self._log_form_completion(instance, submission_type)
response = self._get_open_rosa_response(instance, **openrosa_kwargs)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
def _log_form_details(self, form):
attachments = form.attachments if hasattr(form, 'attachments') else {}
logging.info('Received Form %s with %d attachments',
form.form_id, len(attachments))
for index, (name, attachment) in enumerate(attachments.items()):
attachment_msg = 'Form %s, Attachment %s: %s'
attachment_props = [form.form_id, index, name]
if hasattr(attachment, 'has_size') and attachment.has_size():
attachment_msg = attachment_msg + ' (%d bytes)'
attachment_props.append(attachment.raw_content.size)
logging.info(attachment_msg, *attachment_props)
def _log_form_completion(self, form, submission_type):
# Orig_id doesn't exist on all couch forms, only XFormError and XFormDeprecated
if hasattr(form, 'orig_id') and form.orig_id is not None:
logging.info('Finished %s processing for Form %s with original id %s',
submission_type, form.form_id, form.orig_id)
else:
logging.info('Finished %s processing for Form %s', submission_type, form.form_id)
def _conditionally_send_device_logs_to_sumologic(self, instance):
url = getattr(settings, 'SUMOLOGIC_URL', None)
if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER):
SumoLogicLog(self.domain, instance).send_data(url)
def _invalidate_caches(self, xform):
for device_id in {None, xform.metadata.deviceID if xform.metadata else None}:
self._invalidate_restore_payload_path_cache(xform, device_id)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_restore_task_id_cache(xform, device_id)
def _invalidate_restore_payload_path_cache(self, xform, device_id):
"""invalidate cached initial restores"""
restore_payload_path_cache = RestorePayloadPathCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=xform.last_sync_token,
device_id=device_id,
)
restore_payload_path_cache.invalidate()
def _invalidate_async_restore_task_id_cache(self, xform, device_id):
async_restore_task_id_cache = AsyncRestoreTaskIdCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=self.last_sync_token,
device_id=device_id,
)
task_id = async_restore_task_id_cache.get_value()
if task_id is not None:
revoke_celery_task(task_id)
async_restore_task_id_cache.invalidate()
@tracer.wrap(name='submission.save_models')
def save_processed_models(self, case_db, xforms, case_stock_result):
instance = xforms[0]
try:
with self.timing_context("save_models"), unfinished_submission(instance) as unfinished_submission_stub:
try:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
except PostSaveError:
# mark the stub as saved if there's a post save error
# but re-raise the error so that the re-processing queue picks it up
unfinished_submission_stub.submission_saved()
raise
else:
unfinished_submission_stub.submission_saved()
with self.timing_context("post_save_actions"):
self.do_post_save_actions(case_db, xforms, case_stock_result)
except PostSaveError:
return "Error performing post save operations"
@staticmethod
@tracer.wrap(name='submission.post_save_actions')
def do_post_save_actions(case_db, xforms, case_stock_result):
instance = xforms[0]
case_db.clear_changed()
try:
case_stock_result.stock_result.finalize()
SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models)
close_extension_cases(
case_db,
case_stock_result.case_models,
"SubmissionPost-%s-close_extensions" % instance.form_id
)
except PostSaveError:
raise
except Exception:
notify_exception(get_request(), "Error performing post save actions during form processing", {
'domain': instance.domain,
'form_id': instance.form_id,
})
raise PostSaveError
@staticmethod
@tracer.wrap(name='submission.process_cases_and_stock')
def process_xforms_for_cases(xforms, case_db, timing_context=None):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
timing_context = timing_context or TimingContext()
instance = xforms[0]
with timing_context("process_cases"):
case_result = process_cases_with_casedb(xforms, case_db)
with timing_context("process_ledgers"):
stock_result = process_stock(xforms, case_db)
stock_result.populate_models()
modified_on_date = instance.received_on
if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on:
modified_on_date = instance.edited_on
with timing_context("check_cases_before_save"):
cases = case_db.get_cases_for_saving(modified_on_date)
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
@staticmethod
def _fire_post_save_signals(instance, cases):
from corehq.form_processor.signals import sql_case_post_save
error_message = "Error occurred during form submission post save (%s)"
error_details = {'domain': instance.domain, 'form_id': instance.form_id}
results = successful_form_received.send_robust(None, xform=instance)
has_errors = log_signal_errors(results, error_message, error_details)
for case in cases:
results = sql_case_post_save.send_robust(case.__class__, case=case)
has_errors |= log_signal_errors(results, error_message, error_details)
if has_errors:
raise PostSaveError
def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None):
if self.is_openrosa_version3:
instance_ok = instance.is_normal or instance.is_duplicate
has_error = error_message or error_nature
if instance_ok and not has_error:
response = openrosa_response.get_openarosa_success_response(message=success_message)
else:
error_message = error_message or instance.problem
response = self.get_v3_error_response(error_message, error_nature)
else:
if instance.is_normal:
response = openrosa_response.get_openarosa_success_response()
else:
response = self.get_v2_submit_error_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_v2_submit_error_response(doc):
return OpenRosaResponse(
message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201,
).response()
@staticmethod
def get_v3_error_response(message, nature):
"""Returns a 422(Unprocessable Entity) response
- if nature == 'processing_failure' the mobile device will quarantine this form and not retry it
- any other value of `nature` will result in the form being marked as a failure and retrying
"""
return OpenRosaResponse(
message=message, nature=nature, status=422,
).response()
@staticmethod
def get_exception_response_and_log(msg, error_instance, path):
logging.warning(
msg,
extra={
'submission_path': path,
'form_id': error_instance.form_id,
'error_message': error_instance.problem
}
)
# This are generally badly formed XML resulting from file corruption, encryption errors
# or other errors on the device which can not be recovered from.
# To prevent retries of these errors we submit a 422 response with `processing_failure` nature.
return OpenRosaResponse(
message="There was an error processing the form: %s" % error_instance.problem,
nature=ResponseNature.PROCESSING_FAILURE,
status=422,
).response()
@tracer.wrap(name='submission.handle_system_action')
def handle_system_action(self, form):
handle_system_action(form, self.auth_context)
self.interface.save_processed_models([form])
response = HttpResponse(status=201)
return FormProcessingResult(response, form, [], [], 'system-action')
@tracer.wrap(name='submission.process_device_log')
def process_device_log(self, device_log_form):
self._conditionally_send_device_logs_to_sumologic(device_log_form)
ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.NO_DEVICE_LOG_ENVS
if self.force_logs or not ignore_device_logs:
try:
process_device_log(self.domain, device_log_form, self.force_logs)
except Exception as e:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
e.sentry_capture = False
raise
response = self._get_open_rosa_response(device_log_form)
return FormProcessingResult(response, device_log_form, [], [], 'device-log')
class CaseDbCacheTest(TestCase):
"""
Tests the functionality of the CaseDbCache object
"""
def setUp(self):
self.interface = FormProcessorInterface()
@run_with_all_backends
def testDomainCheck(self):
id = uuid.uuid4().hex
post_case_blocks([
CaseBlock(
create=True, case_id=id,
user_id='some-user'
).as_xml()
], {'domain': 'good-domain'}
)
bad_cache = self.interface.casedb_cache(domain='bad-domain')
try:
bad_cache.get(id)
self.fail('domain security check failed to raise exception')
except IllegalCaseId:
pass
good_cache = self.interface.casedb_cache(domain='good-domain')
case = good_cache.get(id)
self.assertEqual('some-user', case.user_id) # just sanity check it's the right thing
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
cache = CaseDbCacheCouch()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
@run_with_all_backends
def testGetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i), case.dynamic_case_properties()['my_index'])
for id in case_ids:
self.assertTrue(cache.in_cache(id))
@run_with_all_backends
def testSetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for id in case_ids:
cache.set(id, CaseAccessors().get_case(id))
for i, id in enumerate(case_ids):
self.assertTrue(cache.in_cache(id))
case = cache.get(id)
self.assertEqual(str(i), case.dynamic_case_properties()['my_index'])
@run_with_all_backends
def testPopulate(self):
case_ids = _make_some_cases(3)
cache = self.interface.casedb_cache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
cache.populate(case_ids)
for id in case_ids:
self.assertTrue(cache.in_cache(id))
# sanity check
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i), case.dynamic_case_properties()['my_index'])
def testStripHistory(self):
case_ids = _make_some_cases(3)
history_cache = self.interface.casedb_cache()
for i, id in enumerate(case_ids):
self.assertFalse(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
nohistory_cache = self.interface.casedb_cache(strip_history=True)
for i, id in enumerate(case_ids):
self.assertFalse(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
more_case_ids = _make_some_cases(3)
history_cache.populate(more_case_ids)
nohistory_cache.populate(more_case_ids)
for i, id in enumerate(more_case_ids):
self.assertTrue(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
for i, id in enumerate(more_case_ids):
self.assertTrue(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
def test_nowrap(self):
case_ids = _make_some_cases(1)
cache = self.interface.casedb_cache(wrap=False)
case = cache.get(case_ids[0])
self.assertTrue(isinstance(case, dict))
self.assertFalse(isinstance(case, CommCareCase))
class SubmissionPost(object):
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None):
assert domain, "'domain' is required"
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
if case_db:
assert case_db.domain == domain
self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3
self.track_load = form_load_counter("form_submission", domain)
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
xforms[0] = _transform_instance_to_error(self.interface, error, instance)
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message)
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def _get_success_message(self, instance, cases=None):
'''
Formplayer requests get a detailed success message pointing to the form/case affected.
All other requests get a generic message.
Message is formatted with markdown.
'''
if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID:
return ' √ '
messages = []
user = CouchUser.get_by_user_id(instance.user_id)
if not user or not user.is_web_user():
return _('Form successfully saved!')
from corehq.apps.export.views.list import CaseExportListView, FormExportListView
from corehq.apps.export.views.utils import can_view_case_exports, can_view_form_exports
from corehq.apps.reports.views import CaseDataView, FormDataView
form_link = case_link = form_export_link = case_export_link = None
form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory'
if has_permission_to_view_report(user, instance.domain, form_view):
form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id])
case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport'
if cases and has_permission_to_view_report(user, instance.domain, case_view):
if len(cases) == 1:
case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id])
else:
case_link = ", ".join(["[{}]({})".format(
c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id])
) for c in cases])
if can_view_form_exports(user, instance.domain):
form_export_link = reverse(FormExportListView.urlname, args=[instance.domain])
if cases and can_view_case_exports(user, instance.domain):
case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain])
# Start with generic message
messages.append(_('Form successfully saved!'))
# Add link to form/case if possible
if form_link and case_link:
if len(cases) == 1:
messages.append(
_("You submitted [this form]({}), which affected [this case]({}).")
.format(form_link, case_link))
else:
messages.append(
_("You submitted [this form]({}), which affected these cases: {}.")
.format(form_link, case_link))
elif form_link:
messages.append(_("You submitted [this form]({}).").format(form_link))
elif case_link:
if len(cases) == 1:
messages.append(_("Your form affected [this case]({}).").format(case_link))
else:
messages.append(_("Your form affected these cases: {}.").format(case_link))
# Add link to all form/case exports
if form_export_link and case_export_link:
messages.append(
_("Click to export your [case]({}) or [form]({}) data.")
.format(case_export_link, form_export_link))
elif form_export_link:
messages.append(_("Click to export your [form data]({}).").format(form_export_link))
elif case_export_link:
messages.append(_("Click to export your [case data]({}).").format(case_export_link))
return "\n\n".join(messages)
def run(self):
self.track_load()
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json())
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
if submitted_form.xmlns == DEVICE_LOG_XMLNS:
return self.process_device_log(submitted_form)
cases = []
ledgers = []
submission_type = 'unknown'
openrosa_kwargs = {}
with result.get_locked_forms() as xforms:
if len(xforms) > 1:
self.track_load(len(xforms) - 1)
if self.case_db:
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(
domain=self.domain, lock=True, deleted_ok=True,
xforms=xforms, load_src="form_submission",
)
with case_db_cache as case_db:
instance = xforms[0]
if instance.is_duplicate:
with tracer.trace('submission.process_duplicate'):
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
openrosa_kwargs['error_message'] = result.error
if existing_form.is_error:
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
else:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms,
case_stock_result)
if openrosa_kwargs['error_message']:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases)
elif instance.is_error:
submission_type = 'error'
response = self._get_open_rosa_response(instance, **openrosa_kwargs)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
def _conditionally_send_device_logs_to_sumologic(self, instance):
url = getattr(settings, 'SUMOLOGIC_URL', None)
if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER):
SumoLogicLog(self.domain, instance).send_data(url)
def _invalidate_caches(self, xform):
for device_id in {None, xform.metadata.deviceID if xform.metadata else None}:
self._invalidate_restore_payload_path_cache(xform, device_id)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_restore_task_id_cache(xform, device_id)
def _invalidate_restore_payload_path_cache(self, xform, device_id):
"""invalidate cached initial restores"""
restore_payload_path_cache = RestorePayloadPathCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=xform.last_sync_token,
device_id=device_id,
)
restore_payload_path_cache.invalidate()
def _invalidate_async_restore_task_id_cache(self, xform, device_id):
async_restore_task_id_cache = AsyncRestoreTaskIdCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=self.last_sync_token,
device_id=device_id,
)
task_id = async_restore_task_id_cache.get_value()
if task_id is not None:
revoke_celery_task(task_id)
async_restore_task_id_cache.invalidate()
@tracer.wrap(name='submission.save_models')
def save_processed_models(self, case_db, xforms, case_stock_result):
instance = xforms[0]
try:
with unfinished_submission(instance) as unfinished_submission_stub:
try:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
except PostSaveError:
# mark the stub as saved if there's a post save error
# but re-raise the error so that the re-processing queue picks it up
unfinished_submission_stub.submission_saved()
raise
else:
unfinished_submission_stub.submission_saved()
self.do_post_save_actions(case_db, xforms, case_stock_result)
except PostSaveError:
return "Error performing post save operations"
@staticmethod
@tracer.wrap(name='submission.post_save_actions')
def do_post_save_actions(case_db, xforms, case_stock_result):
instance = xforms[0]
try:
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models)
case_stock_result.case_result.close_extensions(
case_db,
"SubmissionPost-%s-close_extensions" % instance.form_id
)
except PostSaveError:
raise
except Exception:
notify_exception(get_request(), "Error performing post save actions during form processing", {
'domain': instance.domain,
'form_id': instance.form_id,
})
raise PostSaveError
@staticmethod
@tracer.wrap(name='submission.process_cases_and_stock')
def process_xforms_for_cases(xforms, case_db):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
modified_on_date = instance.received_on
if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on:
modified_on_date = instance.edited_on
cases = case_db.get_cases_for_saving(modified_on_date)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
@staticmethod
def _fire_post_save_signals(instance, cases):
from casexml.apps.case.signals import case_post_save
error_message = "Error occurred during form submission post save (%s)"
error_details = {'domain': instance.domain, 'form_id': instance.form_id}
results = successful_form_received.send_robust(None, xform=instance)
has_errors = log_signal_errors(results, error_message, error_details)
for case in cases:
results = case_post_save.send_robust(case.__class__, case=case)
has_errors |= log_signal_errors(results, error_message, error_details)
if has_errors:
raise PostSaveError
def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None):
if self.is_openrosa_version3:
instance_ok = instance.is_normal or instance.is_duplicate
has_error = error_message or error_nature
if instance_ok and not has_error:
response = openrosa_response.get_openarosa_success_response(message=success_message)
else:
error_message = error_message or instance.problem
response = self.get_retry_response(error_message, error_nature)
else:
if instance.is_normal:
response = openrosa_response.get_openarosa_success_response()
else:
response = self.get_v2_submit_error_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_v2_submit_error_response(doc):
return OpenRosaResponse(
message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201,
).response()
@staticmethod
def get_retry_response(message, nature):
"""Returns a 422(Unprocessable Entity) response, mobile will retry this submission
"""
return OpenRosaResponse(
message=message, nature=nature, status=422,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
"Problem receiving submission to %s. Doc id: %s, Error %s" % (
path,
error_instance.form_id,
error_instance.problem
)
)
return OpenRosaResponse(
message="There was an error processing the form: %s" % error_instance.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
@tracer.wrap(name='submission.process_device_log')
def process_device_log(self, device_log_form):
self._conditionally_send_device_logs_to_sumologic(device_log_form)
ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.NO_DEVICE_LOG_ENVS
if not ignore_device_logs:
try:
process_device_log(self.domain, device_log_form)
except Exception as e:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
e.sentry_capture = False
raise
response = self._get_open_rosa_response(device_log_form)
return FormProcessingResult(response, device_log_form, [], [], 'device-log')
class SubmissionPost(object):
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.auth_context = self.auth_context.to_json()
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
xforms[0] = _transform_instance_to_error(self.interface, error, instance)
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message)
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def run(self):
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
submission_type = None
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
cases = []
ledgers = []
submission_type = 'unknown'
response_nature = error_message = None
with result.get_locked_forms() as xforms:
from casexml.apps.case.xform import get_and_check_xform_domain
domain = get_and_check_xform_domain(xforms[0])
if self.case_db:
assert self.case_db.domain == domain
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms)
with case_db_cache as case_db:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
submission_type = 'device_log'
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
error_message = result.error
if existing_form.is_error:
response_nature = ResponseNature.PROCESSING_FAILURE
else:
response_nature = ResponseNature.POST_PROCESSING_FAILIRE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
response_nature = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
error_message = self.save_processed_models(case_db, xforms, case_stock_result)
if error_message:
response_nature = ResponseNature.POST_PROCESSING_FAILIRE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
elif instance.is_error:
submission_type = 'error'
response = self._get_open_rosa_response(instance, error_message, response_nature)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
def _invalidate_caches(self, xform):
for device_id in {None, xform.metadata.deviceID if xform.metadata else None}:
self._invalidate_restore_payload_path_cache(xform, device_id)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_restore_task_id_cache(xform, device_id)
def _invalidate_restore_payload_path_cache(self, xform, device_id):
"""invalidate cached initial restores"""
restore_payload_path_cache = RestorePayloadPathCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=xform.last_sync_token,
device_id=device_id,
)
restore_payload_path_cache.invalidate()
def _invalidate_async_restore_task_id_cache(self, xform, device_id):
async_restore_task_id_cache = AsyncRestoreTaskIdCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=self.last_sync_token,
device_id=device_id,
)
task_id = async_restore_task_id_cache.get_value()
if task_id is not None:
revoke_celery_task(task_id)
async_restore_task_id_cache.invalidate()
def save_processed_models(self, case_db, xforms, case_stock_result):
instance = xforms[0]
try:
with unfinished_submission(instance) as unfinished_submission_stub:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
if unfinished_submission_stub:
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
self.do_post_save_actions(case_db, xforms, case_stock_result)
except PostSaveError:
return "Error performing post save operations"
@staticmethod
def do_post_save_actions(case_db, xforms, case_stock_result):
instance = xforms[0]
try:
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models)
case_stock_result.case_result.close_extensions(
case_db,
"SubmissionPost-%s-close_extensions" % instance.form_id
)
except PostSaveError:
raise
except Exception:
notify_exception(get_request(), "Error performing post save actions during form processing", {
'domain': instance.domain,
'form_id': instance.form_id,
})
raise PostSaveError
@staticmethod
def process_xforms_for_cases(xforms, case_db):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
modified_on_date = instance.received_on
if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on:
modified_on_date = instance.edited_on
cases = case_db.get_cases_for_saving(modified_on_date)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
@staticmethod
def _fire_post_save_signals(instance, cases):
from casexml.apps.case.signals import case_post_save
error_message = "Error occurred during form submission post save (%s)"
error_details = {'domain': instance.domain, 'form_id': instance.form_id}
results = successful_form_received.send_robust(None, xform=instance)
has_errors = log_signal_errors(results, error_message, error_details)
for case in cases:
results = case_post_save.send_robust(case.__class__, case=case)
has_errors |= log_signal_errors(results, error_message, error_details)
if has_errors:
raise PostSaveError
def _get_open_rosa_response(self, instance, error_message=None, error_nature=None):
if self.is_openrosa_version3:
instance_ok = instance.is_normal or instance.is_duplicate
has_error = error_message or error_nature
if instance_ok and not has_error:
response = openrosa_response.SUCCESS_RESPONSE
else:
error_message = error_message or instance.problem
response = self.get_retry_response(error_message, error_nature)
else:
if instance.is_normal:
response = openrosa_response.SUCCESS_RESPONSE
else:
response = self.get_v2_submit_error_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_v2_submit_error_response(doc):
return OpenRosaResponse(
message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201,
).response()
@staticmethod
def get_retry_response(message, nature):
"""Returns a 422(Unprocessable Entity) response, mobile will retry this submission
"""
return OpenRosaResponse(
message=message, nature=nature, status=422,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
u"Problem receiving submission to %s. Doc id: %s, Error %s" % (
path,
error_instance.form_id,
error_instance.problem
)
)
return OpenRosaResponse(
message="There was an error processing the form: %s" % error_instance.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
class SubmissionPost(object):
failed_auth_response = HttpResponseForbidden('Bad auth')
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.auth_context = self.auth_context.to_json()
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
instance = _transform_instance_to_error(self.interface, error, instance)
xforms[0] = instance
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if timezone_migration_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503), None, []
if not self.auth_context.is_valid():
return self.failed_auth_response, None, []
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message), None, []
def _post_process_form(self, xform):
self._set_submission_properties(xform)
if xform.is_submission_error_log:
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def run(self):
failure_result = self._handle_basic_failure_modes()
if failure_result:
return failure_result
result = process_xform_xml(self.domain, self.instance, self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return response, None, []
cases = []
with result.get_locked_forms() as xforms:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
self.interface.save_processed_models([instance])
elif not instance.is_error:
try:
case_stock_result = self.process_xforms_for_cases(xforms)
except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError) as e:
self._handle_known_error(e, instance, xforms)
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
self.save_processed_models(xforms, case_stock_result)
cases = case_stock_result.case_models
errors = self.process_signals(instance)
response = self._get_open_rosa_response(instance, errors)
return response, instance, cases
def save_processed_models(self, xforms, case_stock_result):
from casexml.apps.case.signals import case_post_save
instance = xforms[0]
with unfinished_submission(instance) as unfinished_submission_stub:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
for case in case_stock_result.case_models:
case_post_save.send(case.__class__, case=case)
case_stock_result.case_result.close_extensions()
def process_xforms_for_cases(self, xforms):
from casexml.apps.case.xform import get_and_check_xform_domain
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
domain = get_and_check_xform_domain(instance)
with self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db:
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
cases = case_db.get_cases_for_saving(instance.received_on)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
response, _, _ = self.run()
return response
def process_signals(self, instance):
feedback = successful_form_received.send_robust(None, xform=instance)
errors = []
for func, resp in feedback:
if resp and isinstance(resp, Exception):
error_message = unicode(resp)
logging.error((
u"Receiver app: problem sending "
u"post-save signal %s for xform %s: %s: %s"
) % (func, instance.form_id, type(resp).__name__, error_message))
errors.append(error_message)
if errors:
self.interface.xformerror_from_xform_instance(instance, ", ".join(errors))
self.formdb.update_form_problem_and_state(instance)
return errors
@staticmethod
def get_failed_auth_response():
return HttpResponseForbidden('Bad auth')
def _get_open_rosa_response(self, instance, errors):
if instance.is_normal:
response = self.get_success_response(instance, errors)
else:
response = self.get_failure_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_success_response(doc, errors):
if errors:
response = OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
else:
response = OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u' √ ',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
return response
@staticmethod
def get_failure_response(doc):
return OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
u"Problem receiving submission to %s. Doc id: %s, Error %s" % (
path,
error_instance.form_id,
error_instance.problem
)
)
return OpenRosaResponse(
message=("The sever got itself into big trouble! "
"Details: %s" % error_instance.problem),
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
