class SubmissionPost(object):
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.auth_context = self.auth_context.to_json()
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
instance = _transform_instance_to_error(self.interface, error, instance)
xforms[0] = instance
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message)
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def run(self):
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form.user_id)
submission_type = None
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
cases = []
ledgers = []
submission_type = 'unknown'
with result.get_locked_forms() as xforms:
from casexml.apps.case.xform import get_and_check_xform_domain
domain = get_and_check_xform_domain(xforms[0])
if self.case_db:
assert self.case_db.domain == domain
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms)
with case_db_cache as case_db:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
submission_type = 'device_log'
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
submission_type = 'duplicate'
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
self.save_processed_models(xforms, case_stock_result)
case_stock_result.case_result.close_extensions(case_db)
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
elif instance.is_error:
submission_type = 'error'
errors = self.process_signals(instance)
response = self._get_open_rosa_response(instance, errors)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
@property
def _cache(self):
return get_redis_default_cache()
@property
def _restore_cache_key(self):
from casexml.apps.phone.restore import restore_cache_key
return restore_cache_key
def _invalidate_caches(self, user_id):
"""invalidate cached initial restores"""
initial_restore_cache_key = self._restore_cache_key(
self.domain,
RESTORE_CACHE_KEY_PREFIX,
user_id,
version=V2
)
self._cache.delete(initial_restore_cache_key)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_caches(user_id)
def _invalidate_async_caches(self, user_id):
cache_key = self._restore_cache_key(self.domain, ASYNC_RESTORE_CACHE_KEY_PREFIX, user_id)
task_id = self._cache.get(cache_key)
if task_id is not None:
revoke_celery_task(task_id)
self._cache.delete(cache_key)
def save_processed_models(self, xforms, case_stock_result):
from casexml.apps.case.signals import case_post_save
instance = xforms[0]
with unfinished_submission(instance) as unfinished_submission_stub:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
for case in case_stock_result.case_models:
case_post_save.send(case.__class__, case=case)
@staticmethod
def process_xforms_for_cases(xforms, case_db):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
modified_on_date = instance.received_on
if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on:
modified_on_date = instance.edited_on
cases = case_db.get_cases_for_saving(modified_on_date)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
def process_signals(self, instance):
feedback = successful_form_received.send_robust(None, xform=instance)
errors = []
for func, resp in feedback:
if resp and isinstance(resp, Exception):
error_message = unicode(resp)
logging.error((
u"Receiver app: problem sending "
u"post-save signal %s for xform %s: %s: %s"
) % (func, instance.form_id, type(resp).__name__, error_message))
errors.append(error_message)
if errors:
self.interface.xformerror_from_xform_instance(instance, ", ".join(errors))
self.formdb.update_form_problem_and_state(instance)
return errors
def _get_open_rosa_response(self, instance, errors):
if instance.is_normal and not errors:
response = self.get_success_response()
else:
response = self.get_failure_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_success_response():
return OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u' √ ',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
@staticmethod
def submission_ignored_response():
return OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u'√ (this submission was ignored)',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
@staticmethod
def get_failure_response(doc):
return OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
u"Problem receiving submission to %s. Doc id: %s, Error %s" % (
path,
error_instance.form_id,
error_instance.problem
)
)
return OpenRosaResponse(
message="There was an error processing the form: %s" % error_instance.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
@staticmethod
def get_blacklisted_response():
return OpenRosaResponse(
message=("This submission was blocked because of an unusual volume "
"of submissions from this project space. Please contact "
"support to resolve."),
nature=ResponseNature.SUBMIT_ERROR,
status=509,
).response()
class SubmissionPost(object):
failed_auth_response = HttpResponseForbidden('Bad auth')
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.auth_context = self.auth_context.to_json()
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
instance = _transform_instance_to_error(self.interface, error, instance)
xforms[0] = instance
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if timezone_migration_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503), None, []
if not self.auth_context.is_valid():
return self.failed_auth_response, None, []
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message), None, []
def _post_process_form(self, xform):
self._set_submission_properties(xform)
if xform.is_submission_error_log:
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def run(self):
failure_result = self._handle_basic_failure_modes()
if failure_result:
return failure_result
result = process_xform_xml(self.domain, self.instance, self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return response, None, []
cases = []
with result.get_locked_forms() as xforms:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
self.interface.save_processed_models([instance])
elif not instance.is_error:
try:
case_stock_result = self.process_xforms_for_cases(xforms)
except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError) as e:
self._handle_known_error(e, instance, xforms)
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
self.save_processed_models(xforms, case_stock_result)
cases = case_stock_result.case_models
errors = self.process_signals(instance)
response = self._get_open_rosa_response(instance, errors)
return response, instance, cases
def save_processed_models(self, xforms, case_stock_result):
from casexml.apps.case.signals import case_post_save
instance = xforms[0]
with unfinished_submission(instance) as unfinished_submission_stub:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
for case in case_stock_result.case_models:
case_post_save.send(case.__class__, case=case)
case_stock_result.case_result.close_extensions()
def process_xforms_for_cases(self, xforms):
from casexml.apps.case.xform import get_and_check_xform_domain
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
domain = get_and_check_xform_domain(instance)
with self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db:
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
cases = case_db.get_cases_for_saving(instance.received_on)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
response, _, _ = self.run()
return response
def process_signals(self, instance):
feedback = successful_form_received.send_robust(None, xform=instance)
errors = []
for func, resp in feedback:
if resp and isinstance(resp, Exception):
error_message = unicode(resp)
logging.error((
u"Receiver app: problem sending "
u"post-save signal %s for xform %s: %s: %s"
) % (func, instance.form_id, type(resp).__name__, error_message))
errors.append(error_message)
if errors:
self.interface.xformerror_from_xform_instance(instance, ", ".join(errors))
self.formdb.update_form_problem_and_state(instance)
return errors
@staticmethod
def get_failed_auth_response():
return HttpResponseForbidden('Bad auth')
def _get_open_rosa_response(self, instance, errors):
if instance.is_normal:
response = self.get_success_response(instance, errors)
else:
response = self.get_failure_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_success_response(doc, errors):
if errors:
response = OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
else:
response = OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u' √ ',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
return response
@staticmethod
def get_failure_response(doc):
return OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
u"Problem receiving submission to %s. Doc id: %s, Error %s" % (
path,
error_instance.form_id,
error_instance.problem
)
)
return OpenRosaResponse(
message=("The sever got itself into big trouble! "
"Details: %s" % error_instance.problem),
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
def reprocess_form(form, save=True, lock_form=True):
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(
form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases(
[form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
error_message = '{}: {}'.format(
type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(
form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
else:
def reprocess_form(form, save=True, lock_form=True):
if lock_form:
# track load if locking; otherise it will be tracked elsewhere
form_load_counter("reprocess_form", form.domain)()
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form],
load_src="reprocess_form",
)
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
error_message = '{}: {}'.format(type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id, detail, lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s', ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
else:
if save:
interface.processor.save_processed_models([form], cases, stock_result)
from casexml.apps.stock.models import StockTransaction
ledgers = [
model
for model in stock_result.models_to_save
if isinstance(model, StockTransaction)
]
for ledger in ledgers:
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
return ReprocessingResult(form, cases, ledgers, None)
