def submit_form_locally(instance, domain, max_wait=..., **kwargs):
"""
:param instance: XML instance (as a string) to submit
:param domain: The domain to submit the form to
:param max_wait: Maximum time (in seconds) to allow the process to be delayed if
the project is over its submission rate limit.
The value None means "do not throttle".
The special value ... (Ellipsis) means "The caller did not pass in a value".
(This was chosen because of the special meaning already assumed by None.)
"""
if max_wait is ...:
max_wait = 0.1
if max_wait is not None:
rate_limit_submission(domain,
delay_rather_than_reject=True,
max_wait=max_wait)
# intentionally leave these unauth'd for now
kwargs['auth_context'] = kwargs.get('auth_context') or DefaultAuthContext()
result = SubmissionPost(domain=domain, instance=instance, **kwargs).run()
if not 200 <= result.response.status_code < 300:
raise LocalSubmissionError('Error submitting (status code %s): %s' % (
result.response.status_code,
result.response.content.decode('utf-8', errors='backslashreplace'),
))
return result
def submit_form_locally(instance, domain, **kwargs):
# intentionally leave these unauth'd for now
kwargs['auth_context'] = kwargs.get('auth_context') or DefaultAuthContext()
result = SubmissionPost(domain=domain, instance=instance, **kwargs).run()
if not 200 <= result.response.status_code < 300:
raise LocalSubmissionError('Error submitting (status code %s): %s' % (
result.response.status_code,
result.response.content,
))
return result
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {'backend': 'sql', 'domain': domain}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except Exception:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
# the order of these exceptions is relevant
except UnprocessableFormSubmission as e:
return openrosa_response.OpenRosaResponse(
message=e.message,
nature=openrosa_response.ResponseNature.PROCESSING_FAILURE,
status=e.status_code,
).response()
except BadSubmissionRequest as e:
response = HttpResponse(e.message, status=e.status_code)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
timing_context=timer)
try:
result = submission_post.run()
except XFormLockError as err:
logging.warning('Unable to get lock for form %s', err)
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
response.request_timer = timer # logged as Sentry breadcrumbs in LogLongRequestMiddleware
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = [
"domain:{}".format(domain),
"app_id:{}".format(app_id),
"user_id:{}".format(user_id),
"authenticated:{}".format(authenticated),
"form_meta:{}".format(meta),
]
datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details)
notify_exception(request, "Received a submission with POST.keys()",
details)
response = HttpResponseBadRequest(six.text_type(e))
_record_metrics(metric_tags, 'unknown', response)
return response
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
result = submission_post.run()
response = result.response
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
_record_metrics(metric_tags, result.submission_type, response, result,
timer)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {
'backend': 'sql' if should_use_sql_backend(domain) else 'couch',
'domain': domain
}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
if isinstance(instance, BadRequest):
response = HttpResponseBadRequest(instance.message)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
)
try:
result = submission_post.run()
except XFormLockError as err:
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(request, "Received a submission with POST.keys()",
details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
with TimingContext() as timer:
result = submission_post.run()
response = result.response
tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
u'domain:{}'.format(domain)
]
datadog_counter('commcare.xform_submissions.count',
tags=tags +
['status_code:{}'.format(response.status_code)])
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
elif response.status_code == 201:
datadog_gauge('commcare.xform_submissions.timings',
timer.duration,
tags=tags)
# normalize over number of items (form or case) saved
normalized_time = timer.duration / (1 + len(result.cases))
datadog_gauge('commcare.xform_submissions.normalized_timings',
normalized_time,
tags=tags)
datadog_counter('commcare.xform_submissions.case_count',
len(result.cases),
tags=tags)
datadog_counter('commcare.xform_submissions.ledger_count',
len(result.ledgers),
tags=tags)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
rate_limit_submission_by_delaying(domain, max_wait=15)
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request,
"Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=bool(request.GET.get('force_logs', False)),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request,
"XFormLockError: %s" % err,
XFORM_LOCKED_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
