def get_form_ready_to_save(metadata, is_db_test=False):
from corehq.form_processor.parsers.form import process_xform_xml
from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.models import Attachment
assert metadata is not None
metadata.domain = metadata.domain or uuid.uuid4().hex
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata)
if is_db_test:
wrapped_form = process_xform_xml(metadata.domain,
form_xml).submitted_form
else:
interface = FormProcessorInterface(domain=metadata.domain)
form_json = convert_xform_to_json(form_xml)
wrapped_form = interface.new_xform(form_json)
wrapped_form.domain = metadata.domain
interface.store_attachments(wrapped_form, [
Attachment(
name='form.xml', raw_content=form_xml, content_type='text/xml')
])
wrapped_form.received_on = metadata.received_on
wrapped_form.app_id = metadata.app_id
return wrapped_form
def test(self):
form = _make_form_from_case_blocks([ElementTree.fromstring(CASE_BLOCK)])
result = process_xform_xml(self.domain, form)
with result.get_locked_forms() as xforms:
with self.interface.casedb_cache(domain=self.domain) as case_db:
with self.assertRaises(PhoneDateValueError):
process_cases_with_casedb(xforms, case_db)
def test(self):
form = make_form_from_case_blocks([ElementTree.fromstring(CASE_BLOCK)])
result = process_xform_xml(self.domain, form)
with result.get_locked_forms() as xforms:
with self.interface.casedb_cache(domain=self.domain) as case_db:
with self.assertRaises(PhoneDateValueError):
process_cases_with_casedb(xforms, case_db)
def post_xform(instance_xml, attachments=None, domain='test-domain'):
"""
create a new xform and releases the lock
this is a testing entry point only and is not to be used in real code
"""
result = process_xform_xml(domain, instance_xml, attachments=attachments)
with result.get_locked_forms() as xforms:
FormProcessorInterface(domain).save_processed_models(xforms)
return xforms[0]
def post_xform(instance_xml, attachments=None, domain='test-domain'):
"""
create a new xform and releases the lock
this is a testing entry point only and is not to be used in real code
"""
result = process_xform_xml(domain, instance_xml, attachments=attachments)
with result.get_locked_forms() as xforms:
FormProcessorInterface(domain).save_processed_models(xforms)
return xforms[0]
def run(self):
failure_result = self._handle_basic_failure_modes()
if failure_result:
return failure_result
result = process_xform_xml(self.domain, self.instance, self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form.user_id)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return response, None, []
cases = []
with result.get_locked_forms() as xforms:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
self.interface.save_processed_models([instance])
elif not instance.is_error:
try:
case_stock_result = self.process_xforms_for_cases(xforms)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex) as e:
self._handle_known_error(e, instance, xforms)
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
self.save_processed_models(xforms, case_stock_result)
cases = case_stock_result.case_models
errors = self.process_signals(instance)
response = self._get_open_rosa_response(instance, errors)
return response, instance, cases
def get_form_ready_to_save(metadata, is_db_test=False):
from corehq.form_processor.parsers.form import process_xform_xml
from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json
from corehq.form_processor.interfaces.processor import FormProcessorInterface
assert metadata is not None
metadata.domain = metadata.domain or uuid.uuid4().hex
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata)
if is_db_test:
wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form
else:
form_json = convert_xform_to_json(form_xml)
wrapped_form = FormProcessorInterface(domain=metadata.domain).new_xform(form_json)
wrapped_form.domain = metadata.domain
wrapped_form.received_on = metadata.received_on
return wrapped_form
def get_form_ready_to_save(metadata, is_db_test=False):
from corehq.form_processor.parsers.form import process_xform_xml
from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.models import Attachment
assert metadata is not None
metadata.domain = metadata.domain or uuid.uuid4().hex
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata)
if is_db_test:
wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form
else:
interface = FormProcessorInterface(domain=metadata.domain)
form_json = convert_xform_to_json(form_xml)
wrapped_form = interface.new_xform(form_json)
wrapped_form.domain = metadata.domain
interface.store_attachments(wrapped_form, [
Attachment(name='form.xml', raw_content=form_xml, content_type='text/xml')
])
wrapped_form.received_on = metadata.received_on
wrapped_form.app_id = metadata.app_id
return wrapped_form
def run(self):
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
submission_type = None
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
cases = []
ledgers = []
submission_type = 'unknown'
response_nature = error_message = None
with result.get_locked_forms() as xforms:
from casexml.apps.case.xform import get_and_check_xform_domain
domain = get_and_check_xform_domain(xforms[0])
if self.case_db:
assert self.case_db.domain == domain
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms)
with case_db_cache as case_db:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
submission_type = 'device_log'
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
error_message = result.error
if existing_form.is_error:
response_nature = ResponseNature.PROCESSING_FAILURE
else:
response_nature = ResponseNature.POST_PROCESSING_FAILIRE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
response_nature = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
error_message = self.save_processed_models(case_db, xforms, case_stock_result)
if error_message:
response_nature = ResponseNature.POST_PROCESSING_FAILIRE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
elif instance.is_error:
submission_type = 'error'
response = self._get_open_rosa_response(instance, error_message, response_nature)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
def run(self):
self.track_load()
with self.timing_context("process_xml"):
report_submission_usage(self.domain)
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [],
'known_failures')
result = process_xform_xml(self.domain, self.instance,
self.attachments,
self.auth_context.to_json())
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
if submitted_form.is_submission_error_log:
logging.info('Processing form %s as a submission error',
submitted_form.form_id)
self.formdb.save_new_form(submitted_form)
response = None
try:
xml = self.instance.decode()
except UnicodeDecodeError:
pass
else:
if 'log_subreport' in xml:
response = self.get_exception_response_and_log(
'Badly formed device log', submitted_form,
self.path)
if not response:
response = self.get_exception_response_and_log(
'Problem receiving submission', submitted_form,
self.path)
return FormProcessingResult(response, None, [], [],
'submission_error_log')
if submitted_form.xmlns == SYSTEM_ACTION_XMLNS:
logging.info('Processing form %s as a system action',
submitted_form.form_id)
with self.timing_context("process_system_action"):
return self.handle_system_action(submitted_form)
if submitted_form.xmlns == DEVICE_LOG_XMLNS:
logging.info('Processing form %s as a device log',
submitted_form.form_id)
with self.timing_context("process_device_log"):
return self.process_device_log(submitted_form)
# Begin Normal Form Processing
self._log_form_details(submitted_form)
cases = []
ledgers = []
submission_type = 'unknown'
openrosa_kwargs = {}
with result.get_locked_forms() as xforms:
if len(xforms) > 1:
self.track_load(len(xforms) - 1)
if self.case_db:
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(
domain=self.domain,
lock=True,
deleted_ok=True,
xforms=xforms,
load_src="form_submission",
)
with case_db_cache as case_db:
instance = xforms[0]
if instance.is_duplicate:
with self.timing_context(
"process_duplicate"), tracer.trace(
'submission.process_duplicate'):
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(
stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form,
lock_form=False)
if result and result.error:
submission_type = 'error'
openrosa_kwargs['error_message'] = result.error
if existing_form.is_error:
openrosa_kwargs[
'error_nature'] = ResponseNature.PROCESSING_FAILURE
else:
openrosa_kwargs[
'error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(
xforms, case_db, self.timing_context)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
openrosa_kwargs[
'error_nature'] = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
openrosa_kwargs[
'error_message'] = self.save_processed_models(
case_db, xforms, case_stock_result)
if openrosa_kwargs['error_message']:
openrosa_kwargs[
'error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
openrosa_kwargs[
'success_message'] = self._get_success_message(
instance, cases=cases)
elif instance.is_error:
submission_type = 'error'
self._log_form_completion(instance, submission_type)
response = self._get_open_rosa_response(instance,
**openrosa_kwargs)
return FormProcessingResult(response, instance, cases, ledgers,
submission_type)
def run(self):
self.track_load()
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json())
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
if submitted_form.xmlns == DEVICE_LOG_XMLNS:
return self.process_device_log(submitted_form)
cases = []
ledgers = []
submission_type = 'unknown'
openrosa_kwargs = {}
with result.get_locked_forms() as xforms:
if len(xforms) > 1:
self.track_load(len(xforms) - 1)
if self.case_db:
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(
domain=self.domain, lock=True, deleted_ok=True,
xforms=xforms, load_src="form_submission",
)
with case_db_cache as case_db:
instance = xforms[0]
if instance.is_duplicate:
with tracer.trace('submission.process_duplicate'):
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
openrosa_kwargs['error_message'] = result.error
if existing_form.is_error:
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
else:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms,
case_stock_result)
if openrosa_kwargs['error_message']:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases)
elif instance.is_error:
submission_type = 'error'
response = self._get_open_rosa_response(instance, **openrosa_kwargs)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
def run(self):
self.track_load()
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json())
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(submitted_form, self.path)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
if submitted_form.xmlns == DEVICE_LOG_XMLNS:
return self.process_device_log(submitted_form)
cases = []
ledgers = []
submission_type = 'unknown'
openrosa_kwargs = {}
with result.get_locked_forms() as xforms:
if len(xforms) > 1:
self.track_load(len(xforms) - 1)
if self.case_db:
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(
domain=self.domain, lock=True, deleted_ok=True,
xforms=xforms, load_src="form_submission",
)
with case_db_cache as case_db:
instance = xforms[0]
is_edit_of_error = len(xforms) == 2 and xforms[1].is_error
if not instance.is_duplicate and is_edit_of_error:
# edge case from ICDS where a form errors and then future re-submissions of the same
# form do not have the same MD5 hash due to a bug on mobile:
# see https://dimagi-dev.atlassian.net/browse/ICDS-376
existing_form = xforms[1]
if not existing_form.initial_processing_complete:
# since we have a new form and the old one was not successfully processed
# we can effectively ignore this form and process the new one as normal
# since this form will have been marked as deprecated and have a new ID etc.
existing_form.save()
xforms = [instance]
else:
# not clear what to do in this case, try the normal edit workflow
pass
if instance.is_duplicate:
with tracer.trace('submission.process_duplicate'):
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
openrosa_kwargs['error_message'] = result.error
if existing_form.is_error:
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
else:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms,
case_stock_result)
if openrosa_kwargs['error_message']:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases)
elif instance.is_error:
submission_type = 'error'
response = self._get_open_rosa_response(instance, **openrosa_kwargs)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
