def should_ignore_submission(request):
"""
If IGNORE_ALL_DEMO_USER_SUBMISSIONS is True then ignore submission if from demo user.
Else
If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
the submissions should be ignored
"""
form_json = None
if IGNORE_ALL_DEMO_USER_SUBMISSIONS:
instance, _ = couchforms.get_instance_and_attachment(request)
try:
form_json = convert_xform_to_json(instance)
except couchforms.XMLSyntaxError:
# let the usual workflow handle response for invalid xml
return False
else:
form_meta = form_json.get('meta')
if form_meta and _submitted_by_demo_user(form_meta, request.domain):
_notify_submission_if_applicable(request, form_meta)
return True
if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
return False
if form_json is None:
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
return False if from_demo_user(form_json) else True
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
instance, attachments = couchforms.get_instance_and_attachment(request)
response = couchforms.SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error('Status code 400 for a form submission. '
'See couchlog db for more info: %s' % db_response['id'])
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _noauth_post(request, domain, app_id=None):
"""
This is explictly called for a submission that has secure submissions enabled, but is manually
overriding the submit URL to not specify auth context. It appears to be used by demo mode.
It mainly just checks that we are touching test data only in the right domain and submitting
as demo_user.
"""
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
try:
# require new-style meta/userID (reject Meta/chw_id)
if form_json['meta']['userID'] == 'demo_user':
return True
except (KeyError, ValueError):
pass
if is_device_report(form_json):
return True
return False
def case_block_ok(case_updates):
"""
Check for all cases that we are submitting as demo_user and that the domain we
are submitting against for any previously existing cases matches the submission
domain.
"""
allowed_ids = ('demo_user', 'demo_user_group_id', None)
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in allowed_ids:
return False
if create_action.owner_id not in allowed_ids:
return False
if update_action:
if update_action.owner_id not in allowed_ids:
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
# todo: consider whether we want to remove this call, and/or pass the result
# through to the next function so we don't have to get the cases again later
cases = CommCareCase.bulk_get_lite(list(case_ids))
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id not in allowed_ids:
return False
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
return response
def should_ignore_submission(request):
"""
If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
the submissions should be ignored
"""
if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
return False
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
return False if from_demo_user(form_json) else True
def _noauth_post(request, domain, app_id=None):
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
try:
# require new-style meta/userID (reject Meta/chw_id)
if form_json['meta']['userID'] == 'demo_user':
return True
except (KeyError, ValueError):
pass
if is_device_report(form_json):
return True
return False
def case_block_ok(case_updates):
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in ('demo_user', None):
return False
if create_action.owner_id not in ('demo_user', None):
return False
if update_action:
if update_action.owner_id not in ('demo_user', None):
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
cases = CommCareCase.bulk_get_lite(list(case_ids))
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id != 'demo_user':
return False
return True
if not (form_ok(form_json) and case_block_ok(case_updates)):
return HttpResponseForbidden()
return _process_form(
request=request,
domain=domain,
app_id=app_id,
user_id=None,
authenticated=False,
auth_cls=WaivedAuthContext,
)
def should_ignore_submission(request):
"""
If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
the submissions should be ignored
"""
if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
return False
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
return False if from_demo_user(form_json) else True
def _noauth_post(request, domain, app_id=None):
"""
This is explicitly called for a submission that has secure submissions enabled, but is manually
overriding the submit URL to not specify auth context. It appears to be used by demo mode.
It mainly just checks that we are touching test data only in the right domain and submitting
as demo_user.
"""
try:
instance, _ = couchforms.get_instance_and_attachment(request)
except BadSubmissionRequest as e:
return HttpResponseBadRequest(e.message)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
return (from_demo_user(form_json) or is_device_report(form_json))
def case_block_ok(case_updates):
"""
Check for all cases that we are submitting as demo_user and that the domain we
are submitting against for any previously existing cases matches the submission
domain.
"""
allowed_ids = ('demo_user', 'demo_user_group_id', None)
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in allowed_ids:
return False
if create_action.owner_id not in allowed_ids:
return False
if update_action:
if update_action.owner_id not in allowed_ids:
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
# todo: consider whether we want to remove this call, and/or pass the result
# through to the next function so we don't have to get the cases again later
cases = CommCareCase.objects.get_cases(list(case_ids), domain)
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id not in allowed_ids:
return False
def post(request):
"""
XForms can get posted here. They will be forwarded to couch.
Just like play, if you specify a callback you get called,
otherwise you get a generic response. Callbacks follow
a different signature as play, only passing in the document
(since we don't know what xform was being posted to)
"""
instance, attachments = couchforms.get_instance_and_attachment(request)
return SubmissionPost(
instance=instance,
attachments=attachments,
path=couchforms.get_path(request),
).get_response()
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
instance, attachments = couchforms.get_instance_and_attachment(request)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = couchforms.SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request':
unicode(request),
'response':
unicode(response),
})
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
))
return response
def _noauth_post(request, domain, app_id=None):
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
try:
# require new-style meta/userID (reject Meta/chw_id)
if form_json['meta']['userID'] == 'demo_user':
return True
except (KeyError, ValueError):
pass
if is_device_report(form_json):
return True
return False
def case_block_ok(case_updates):
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in ('demo_user', None):
return False
if create_action.owner_id not in ('demo_user', None):
return False
if update_action:
if update_action.owner_id not in ('demo_user', None):
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
cases = CommCareCase.bulk_get_lite(list(case_ids))
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id != 'demo_user':
return False
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = [
"domain:{}".format(domain),
"app_id:{}".format(app_id),
"user_id:{}".format(user_id),
"authenticated:{}".format(authenticated),
"form_meta:{}".format(meta),
]
datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details)
notify_exception(request, "Received a submission with POST.keys()",
details)
response = HttpResponseBadRequest(six.text_type(e))
_record_metrics(metric_tags, 'unknown', response)
return response
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
result = submission_post.run()
response = result.response
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
_record_metrics(metric_tags, result.submission_type, response, result,
timer)
return response
def _noauth_post(request, domain, app_id=None):
"""
This is explictly called for a submission that has secure submissions enabled, but is manually
overriding the submit URL to not specify auth context. It appears to be used by demo mode.
It mainly just checks that we are touching test data only in the right domain and submitting
as demo_user.
"""
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
try:
# require new-style meta/userID (reject Meta/chw_id)
if form_json['meta']['userID'] == 'demo_user':
return True
except (KeyError, ValueError):
pass
if is_device_report(form_json):
return True
return False
def case_block_ok(case_updates):
"""
Check for all cases that we are submitting as demo_user and that the domain we
are submitting against for any previously existing cases matches the submission
domain.
"""
allowed_ids = ('demo_user', 'demo_user_group_id', None)
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in allowed_ids:
return False
if create_action.owner_id not in allowed_ids:
return False
if update_action:
if update_action.owner_id not in allowed_ids:
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
# todo: consider whether we want to remove this call, and/or pass the result
# through to the next function so we don't have to get the cases again later
cases = CommCareCase.bulk_get_lite(list(case_ids))
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id not in allowed_ids:
return False
return True
if not (form_ok(form_json) and case_block_ok(case_updates)):
return HttpResponseForbidden()
return _process_form(
request=request,
domain=domain,
app_id=app_id,
user_id=None,
authenticated=False,
auth_cls=WaivedAuthContext,
)
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {
'backend': 'sql' if should_use_sql_backend(domain) else 'couch',
'domain': domain
}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
if isinstance(instance, BadRequest):
response = HttpResponseBadRequest(instance.message)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
)
try:
result = submission_post.run()
except XFormLockError as err:
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(request, "Received a submission with POST.keys()",
details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
with TimingContext() as timer:
result = submission_post.run()
response = result.response
tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
u'domain:{}'.format(domain)
]
datadog_counter('commcare.xform_submissions.count',
tags=tags +
['status_code:{}'.format(response.status_code)])
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
elif response.status_code == 201:
datadog_gauge('commcare.xform_submissions.timings',
timer.duration,
tags=tags)
# normalize over number of items (form or case) saved
normalized_time = timer.duration / (1 + len(result.cases))
datadog_gauge('commcare.xform_submissions.normalized_timings',
normalized_time,
tags=tags)
datadog_counter('commcare.xform_submissions.case_count',
len(result.cases),
tags=tags)
datadog_counter('commcare.xform_submissions.ledger_count',
len(result.ledgers),
tags=tags)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request, "Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags,
domain, app_id, user_id, authenticated, meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT,
metric_tags, domain, app_id, user_id, authenticated, status=423,
notify=False,
)
response = result.response
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
_record_metrics(metric_tags, result.submission_type, response, result, timer)
return response
def _noauth_post(request, domain, app_id=None):
"""
This is explictly called for a submission that has secure submissions enabled, but is manually
overriding the submit URL to not specify auth context. It appears to be used by demo mode.
It mainly just checks that we are touching test data only in the right domain and submitting
as demo_user.
"""
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
return (from_demo_user(form_json) or is_device_report(form_json))
def case_block_ok(case_updates):
"""
Check for all cases that we are submitting as demo_user and that the domain we
are submitting against for any previously existing cases matches the submission
domain.
"""
allowed_ids = ('demo_user', 'demo_user_group_id', None)
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in allowed_ids:
return False
if create_action.owner_id not in allowed_ids:
return False
if update_action:
if update_action.owner_id not in allowed_ids:
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
# todo: consider whether we want to remove this call, and/or pass the result
# through to the next function so we don't have to get the cases again later
cases = CaseAccessors(domain).get_cases(list(case_ids))
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id not in allowed_ids:
return False
return True
if not (form_ok(form_json) and case_block_ok(case_updates)):
if request.GET.get('submit_mode') != DEMO_SUBMIT_MODE:
# invalid submissions under demo mode submission can be processed
return HttpResponseForbidden()
return _process_form(
request=request,
domain=domain,
app_id=app_id,
user_id=None,
authenticated=False,
auth_cls=WaivedAuthContext,
)
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {'backend': 'sql', 'domain': domain}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except Exception:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
# the order of these exceptions is relevant
except UnprocessableFormSubmission as e:
return openrosa_response.OpenRosaResponse(
message=e.message,
nature=openrosa_response.ResponseNature.PROCESSING_FAILURE,
status=e.status_code,
).response()
except BadSubmissionRequest as e:
response = HttpResponse(e.message, status=e.status_code)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
timing_context=timer)
try:
result = submission_post.run()
except XFormLockError as err:
logging.warning('Unable to get lock for form %s', err)
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
response.request_timer = timer # logged as Sentry breadcrumbs in LogLongRequestMiddleware
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
rate_limit_submission_by_delaying(domain, max_wait=15)
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request,
"Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=bool(request.GET.get('force_logs', False)),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request,
"XFormLockError: %s" % err,
XFORM_LOCKED_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
