def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
instance, attachments = couchforms.get_instance_and_attachment(request)
response = couchforms.SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error('Status code 400 for a form submission. '
'See couchlog db for more info: %s' % db_response['id'])
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
instance, attachments = couchforms.get_instance_and_attachment(request)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = couchforms.SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request':
unicode(request),
'response':
unicode(response),
})
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
))
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {'backend': 'sql', 'domain': domain}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except Exception:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
# the order of these exceptions is relevant
except UnprocessableFormSubmission as e:
return openrosa_response.OpenRosaResponse(
message=e.message,
nature=openrosa_response.ResponseNature.PROCESSING_FAILURE,
status=e.status_code,
).response()
except BadSubmissionRequest as e:
response = HttpResponse(e.message, status=e.status_code)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
timing_context=timer)
try:
result = submission_post.run()
except XFormLockError as err:
logging.warning('Unable to get lock for form %s', err)
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
response.request_timer = timer # logged as Sentry breadcrumbs in LogLongRequestMiddleware
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = [
"domain:{}".format(domain),
"app_id:{}".format(app_id),
"user_id:{}".format(user_id),
"authenticated:{}".format(authenticated),
"form_meta:{}".format(meta),
]
datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details)
notify_exception(request, "Received a submission with POST.keys()",
details)
response = HttpResponseBadRequest(six.text_type(e))
_record_metrics(metric_tags, 'unknown', response)
return response
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
result = submission_post.run()
response = result.response
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
_record_metrics(metric_tags, result.submission_type, response, result,
timer)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {
'backend': 'sql' if should_use_sql_backend(domain) else 'couch',
'domain': domain
}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
if isinstance(instance, BadRequest):
response = HttpResponseBadRequest(instance.message)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
)
try:
result = submission_post.run()
except XFormLockError as err:
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(request, "Received a submission with POST.keys()",
details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
with TimingContext() as timer:
result = submission_post.run()
response = result.response
tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
u'domain:{}'.format(domain)
]
datadog_counter('commcare.xform_submissions.count',
tags=tags +
['status_code:{}'.format(response.status_code)])
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
elif response.status_code == 201:
datadog_gauge('commcare.xform_submissions.timings',
timer.duration,
tags=tags)
# normalize over number of items (form or case) saved
normalized_time = timer.duration / (1 + len(result.cases))
datadog_gauge('commcare.xform_submissions.normalized_timings',
normalized_time,
tags=tags)
datadog_counter('commcare.xform_submissions.case_count',
len(result.cases),
tags=tags)
datadog_counter('commcare.xform_submissions.ledger_count',
len(result.ledgers),
tags=tags)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request, "Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags,
domain, app_id, user_id, authenticated, meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT,
metric_tags, domain, app_id, user_id, authenticated, status=423,
notify=False,
)
response = result.response
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
_record_metrics(metric_tags, result.submission_type, response, result, timer)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
rate_limit_submission_by_delaying(domain, max_wait=15)
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request,
"Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=bool(request.GET.get('force_logs', False)),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request,
"XFormLockError: %s" % err,
XFORM_LOCKED_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response