def test_submit_mode(self):
# test 'submit_mode=demo' request param
accepted_response = SubmissionPost.get_success_response(None, None).content
ignored_response = SubmissionPost.submission_ignored_response().content
client = django_digest.test.Client()
client.set_authorization(self.user.username, "1234", method="Digest")
# submissions with 'submit_mode=demo' param by real users should be ignored
self._test_post(
file_path=self.simple_form,
client=client,
authtype="digest",
submit_mode=DEMO_SUBMIT_MODE,
expected_response=ignored_response,
)
# submissions with 'submit_mode=demo' param by real users should be ignored
# even if no authorization headers are supplied
self._test_post(
file_path=self.simple_form,
client=client,
authtype="noauth",
submit_mode=DEMO_SUBMIT_MODE,
expected_response=ignored_response,
)
# submissions by 'demo_user' should not be ignored even with 'submit_mode=demo' param
self._test_post(
file_path=self.form_with_demo_case,
authtype="noauth",
submit_mode=DEMO_SUBMIT_MODE,
expected_response=accepted_response,
)
def _perfom_post_save_actions(form, save=True):
interface = FormProcessorInterface(form.domain)
cache = interface.casedb_cache(
domain=form.domain, lock=False, deleted_ok=True, xforms=[form],
load_src="reprocess_form_post_save",
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
case_models = case_stock_result.case_models
if interface.use_sql_domain:
forms = ProcessedForms(form, None)
stock_result = case_stock_result.stock_result
try:
FormProcessorSQL.publish_changes_to_kafka(forms, case_models, stock_result)
except Exception:
error_message = "Error publishing to kafka"
return ReprocessingResult(form, None, None, error_message)
try:
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
except PostSaveError:
error_message = "Error performing post save operations"
return ReprocessingResult(form, None, None, error_message)
return ReprocessingResult(form, case_models, None, None)
def test_submit_mode(self):
# test 'submit_mode=demo' request param
accepted_response = SubmissionPost.get_success_response(None,
None).content
ignored_response = SubmissionPost.submission_ignored_response().content
client = django_digest.test.Client()
client.set_authorization(self.user.username, '1234', method='Digest')
# submissions with 'submit_mode=demo' param by real users should be ignored
self._test_post(file_path=self.simple_form,
client=client,
authtype='digest',
submit_mode=DEMO_SUBMIT_MODE,
expected_response=ignored_response)
# submissions with 'submit_mode=demo' param by real users should be ignored
# even if no authorization headers are supplied
self._test_post(file_path=self.simple_form,
client=client,
authtype='noauth',
submit_mode=DEMO_SUBMIT_MODE,
expected_response=ignored_response)
# submissions by 'demo_user' should not be ignored even with 'submit_mode=demo' param
self._test_post(file_path=self.form_with_demo_case,
authtype='noauth',
submit_mode=DEMO_SUBMIT_MODE,
expected_response=accepted_response)
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
return response
def _perfom_post_save_actions(form, save=True):
interface = FormProcessorInterface(form.domain)
cache = interface.casedb_cache(
domain=form.domain, lock=False, deleted_ok=True, xforms=[form]
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
try:
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
except PostSaveError:
error_message = "Error performing post save operations"
return ReprocessingResult(form, None, None, error_message)
return ReprocessingResult(form, case_stock_result.case_models, None, None)
def submit_form_locally(instance, domain, max_wait=..., **kwargs):
"""
:param instance: XML instance (as a string) to submit
:param domain: The domain to submit the form to
:param max_wait: Maximum time (in seconds) to allow the process to be delayed if
the project is over its submission rate limit.
The value None means "do not throttle".
The special value ... (Ellipsis) means "The caller did not pass in a value".
(This was chosen because of the special meaning already assumed by None.)
"""
if max_wait is ...:
max_wait = 0.1
if max_wait is not None:
rate_limit_submission(domain,
delay_rather_than_reject=True,
max_wait=max_wait)
# intentionally leave these unauth'd for now
kwargs['auth_context'] = kwargs.get('auth_context') or DefaultAuthContext()
result = SubmissionPost(domain=domain, instance=instance, **kwargs).run()
if not 200 <= result.response.status_code < 300:
raise LocalSubmissionError('Error submitting (status code %s): %s' % (
result.response.status_code,
result.response.content.decode('utf-8', errors='backslashreplace'),
))
return result
def test_noauth_demomode(self):
self._test_post(
file_path=self.bare_form,
authtype="noauth",
expected_status=201,
submit_mode=DEMO_SUBMIT_MODE,
expected_response=SubmissionPost.submission_ignored_response().content,
)
def test_noauth_demomode(self):
self._test_post(
file_path=self.bare_form,
authtype='noauth',
expected_status=201,
submit_mode=DEMO_SUBMIT_MODE,
expected_response=SubmissionPost.submission_ignored_response().
content,
)
def submit_form_locally(instance, domain, **kwargs):
# intentionally leave these unauth'd for now
kwargs['auth_context'] = kwargs.get('auth_context') or DefaultAuthContext()
result = SubmissionPost(domain=domain, instance=instance, **kwargs).run()
if not 200 <= result.response.status_code < 300:
raise LocalSubmissionError('Error submitting (status code %s): %s' % (
result.response.status_code,
result.response.content,
))
return result
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {
'backend': 'sql' if should_use_sql_backend(domain) else 'couch',
'domain': domain
}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
if isinstance(instance, BadRequest):
response = HttpResponseBadRequest(instance.message)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
)
try:
result = submission_post.run()
except XFormLockError as err:
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(request, "Received a submission with POST.keys()",
details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
with TimingContext() as timer:
result = submission_post.run()
response = result.response
tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
u'domain:{}'.format(domain)
]
datadog_counter('commcare.xform_submissions.count',
tags=tags +
['status_code:{}'.format(response.status_code)])
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
elif response.status_code == 201:
datadog_gauge('commcare.xform_submissions.timings',
timer.duration,
tags=tags)
# normalize over number of items (form or case) saved
normalized_time = timer.duration / (1 + len(result.cases))
datadog_gauge('commcare.xform_submissions.normalized_timings',
normalized_time,
tags=tags)
datadog_counter('commcare.xform_submissions.case_count',
len(result.cases),
tags=tags)
datadog_counter('commcare.xform_submissions.ledger_count',
len(result.ledgers),
tags=tags)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request, "Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags,
domain, app_id, user_id, authenticated, meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT,
metric_tags, domain, app_id, user_id, authenticated, status=423,
notify=False,
)
response = result.response
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
_record_metrics(metric_tags, result.submission_type, response, result, timer)
return response
def reprocess_form(form, save=True, lock_form=True):
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(
form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases(
[form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
error_message = '{}: {}'.format(
type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(
form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
else:
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = [
"domain:{}".format(domain),
"app_id:{}".format(app_id),
"user_id:{}".format(user_id),
"authenticated:{}".format(authenticated),
"form_meta:{}".format(meta),
]
datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details)
notify_exception(request, "Received a submission with POST.keys()",
details)
response = HttpResponseBadRequest(six.text_type(e))
_record_metrics(metric_tags, 'unknown', response)
return response
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
result = submission_post.run()
response = result.response
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
_record_metrics(metric_tags, result.submission_type, response, result,
timer)
return response
def reprocess_form(form, save=True, lock_form=True):
if lock_form:
# track load if locking; otherise it will be tracked elsewhere
form_load_counter("reprocess_form", form.domain)()
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form],
load_src="reprocess_form",
)
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
error_message = '{}: {}'.format(type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id, detail, lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s', ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
else:
if save:
interface.processor.save_processed_models([form], cases, stock_result)
from casexml.apps.stock.models import StockTransaction
ledgers = [
model
for model in stock_result.models_to_save
if isinstance(model, StockTransaction)
]
for ledger in ledgers:
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
return ReprocessingResult(form, cases, ledgers, None)
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def reprocess_xform_error(form):
"""
Attempt to re-process an error form. This was created specifically to address
the issue of out of order forms and child cases (form creates child case before
parent case has been created).
See http://manage.dimagi.com/default.asp?250459
:param form_id: ID of the error form to process
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.submission_post import SubmissionPost
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL, FormAccessorSQL, LedgerAccessorSQL
from corehq.blobs.mixin import bulk_atomic_blobs
from couchforms.models import XFormInstance
from casexml.apps.case.signals import case_post_save
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
if not form:
raise Exception('Form with ID {} not found'.format(form.form_id))
if not form.is_error:
raise Exception('Form was not an error form: {}={}'.format(form.form_id, form.doc_type))
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
form.initial_processing_complete = True
form.problem = None
cache = FormProcessorInterface(form.domain).casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form]
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
if case_stock_result:
stock_result = case_stock_result.stock_result
if stock_result:
assert stock_result.populated
cases = case_stock_result.case_models
if should_use_sql_backend(form.domain):
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
LedgerAccessorSQL.save_ledger_values(stock_result.models_to_save)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None),
cases,
stock_result
)
else:
with bulk_atomic_blobs([form] + cases):
XFormInstance.save(form) # use this save to that we don't overwrite the doc_type
XFormInstance.get_db().bulk_save(cases)
if stock_result:
stock_result.commit()
case_stock_result.stock_result.finalize()
case_stock_result.case_result.commit_dirtiness_flags()
for case in cases:
case_post_save.send(case.__class__, case=case)
return form
**ledger.ledger_reference._asdict())
else:
if save:
interface.processor.save_processed_models([form], cases,
stock_result)
from casexml.apps.stock.models import StockTransaction
ledgers = [
model for model in stock_result.models_to_save
if isinstance(model, StockTransaction)
]
for ledger in ledgers:
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
save and SubmissionPost.do_post_save_actions(
casedb, [form], case_stock_result)
return ReprocessingResult(form, cases, ledgers, None)
def _log_changes(cases, stock_updates, stock_deletes):
if logger.isEnabledFor(logging.INFO):
case_ids = [case.case_id for case in cases]
logger.info(
"changes:\n\tcases: %s\n\tstock changes%s\n\tstock deletes%s",
case_ids, stock_updates, stock_deletes)
def _get_case_ids_needing_rebuild(form, cases):
"""Return a set of case IDs for cases that have been modified since the form was
originally submitted i.e. are needing to be rebuilt"""
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {'backend': 'sql', 'domain': domain}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except Exception:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
# the order of these exceptions is relevant
except UnprocessableFormSubmission as e:
return openrosa_response.OpenRosaResponse(
message=e.message,
nature=openrosa_response.ResponseNature.PROCESSING_FAILURE,
status=e.status_code,
).response()
except BadSubmissionRequest as e:
response = HttpResponse(e.message, status=e.status_code)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
timing_context=timer)
try:
result = submission_post.run()
except XFormLockError as err:
logging.warning('Unable to get lock for form %s', err)
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
response.request_timer = timer # logged as Sentry breadcrumbs in LogLongRequestMiddleware
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
rate_limit_submission_by_delaying(domain, max_wait=15)
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request,
"Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=bool(request.GET.get('force_logs', False)),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request,
"XFormLockError: %s" % err,
XFORM_LOCKED_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response