def should_ignore_submission(request):
"""
If IGNORE_ALL_DEMO_USER_SUBMISSIONS is True then ignore submission if from demo user.
Else
If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
the submissions should be ignored
"""
form_json = None
if IGNORE_ALL_DEMO_USER_SUBMISSIONS:
instance, _ = couchforms.get_instance_and_attachment(request)
try:
form_json = convert_xform_to_json(instance)
except couchforms.XMLSyntaxError:
# let the usual workflow handle response for invalid xml
return False
else:
form_meta = form_json.get('meta')
if form_meta and _submitted_by_demo_user(form_meta, request.domain):
_notify_submission_if_applicable(request, form_meta)
return True
if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
return False
if form_json is None:
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
return False if from_demo_user(form_json) else True
def get_form_ready_to_save(metadata, is_db_test=False):
from corehq.form_processor.parsers.form import process_xform_xml
from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.models import Attachment
assert metadata is not None
metadata.domain = metadata.domain or uuid.uuid4().hex
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata)
if is_db_test:
wrapped_form = process_xform_xml(metadata.domain,
form_xml).submitted_form
else:
interface = FormProcessorInterface(domain=metadata.domain)
form_json = convert_xform_to_json(form_xml)
wrapped_form = interface.new_xform(form_json)
wrapped_form.domain = metadata.domain
interface.store_attachments(wrapped_form, [
Attachment(
name='form.xml', raw_content=form_xml, content_type='text/xml')
])
wrapped_form.received_on = metadata.received_on
wrapped_form.app_id = metadata.app_id
return wrapped_form
def _get_new_form_json(xml, xform_id):
form_json = convert_xform_to_json(xml)
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_json)
# this is actually in-place because of how jsonobject works
scrub_meta(XFormInstance.wrap({'form': form_json, '_id': xform_id}))
return form_json
def test_single_entry(self):
form_data = convert_xform_to_json(self.get_xml('single_entry'))
self.assertEqual(
_get_logs(form_data, 'user_error_subreport', 'user_error'), [{
"session":
"frame: (COMMAND_ID m1)",
"user_id":
"65t2l8ga654k93z92j236e2h30jt048b",
"expr":
"",
"app_id":
"skj94l95tw0k6v8esdj9s2g4chfpup83",
"version":
"89",
"msg": ("XPath evaluation: type mismatch It looks like this "
"question contains a reference to path number which "
"evaluated to instance(item-list:numbers)/numbers_list"
"/numbers[1]/number which was not found. This often "
"means you forgot to include the full path to the "
"question -- e.g. /data/[node]"),
"@date":
"2016-03-23T12:27:33.681-04",
"type":
"error-config"
}])
def create_case(case_id, files, patient_case_id=None):
"""
Handle case submission for the sonosite endpoint
"""
# we already parsed what we need from this, so can just remove it
# without worrying we will need it later
files.pop('PT_PPS.XML', '')
xform = render_sonosite_xform(files, case_id, patient_case_id)
file_dict = {}
for f in files:
file_dict[f] = UploadedFile(files[f], f)
submit_form_locally(
instance=xform,
attachments=file_dict,
domain=UTH_DOMAIN,
)
# this is a bit of a hack / abstraction violation
# would be nice if submit_form_locally returned info about cases updated
case_ids = {
case_update.id
for case_update in get_case_updates(convert_xform_to_json(xform))
}
return [CommCareCase.get(case_id) for case_id in case_ids]
def _get_new_form_json(xml, xform_id):
form_json = convert_xform_to_json(xml)
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_json)
# this is actually in-place because of how jsonobject works
scrub_meta(XFormInstance.wrap({'form': form_json, '_id': xform_id}))
return form_json
def _get_form_json(self, xml_file):
form_json = convert_xform_to_json(self.get_xml(xml_file))
return {
'form': form_json,
'domain': self.domain,
'xmlns': form_json['@xmlns'],
'doc_type': 'XFormInstance',
}
def _get_form_json(self, xml_file):
form_json = convert_xform_to_json(self.get_xml(xml_file))
return {
'form': form_json,
'domain': self.domain,
'xmlns': form_json['@xmlns'],
'doc_type': 'XFormInstance',
}
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
return response
def test_single_node(self):
form_data = convert_xform_to_json(self.get_xml('single_node'))
self.assertEqual(
_get_logs(form_data, 'log_subreport', 'log'),
[{'@date': '2016-03-20T20:46:08.664+05:30',
'msg': 'Logging out service login',
'type': 'maintenance'},
{'@date': '2016-03-20T20:46:08.988+05:30',
'msg': 'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8',
'type': 'user'}]
)
def test_single_node(self):
form_data = convert_xform_to_json(self.get_xml('single_node'))
self.assertEqual(
_get_logs(form_data, 'log_subreport', 'log'),
[{u'@date': u'2016-03-20T20:46:08.664+05:30',
u'msg': u'Logging out service login',
u'type': u'maintenance'},
{u'@date': u'2016-03-20T20:46:08.988+05:30',
u'msg': u'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8',
u'type': u'user'}]
)
def new_form_from_old(cls, existing_form, xml, value_responses_map, user_id):
from corehq.form_processor.parsers.form import apply_deprecation
new_xml = etree.tostring(xml)
form_json = convert_xform_to_json(new_xml)
new_form = cls.new_xform(form_json)
new_form.user_id = user_id
new_form.domain = existing_form.domain
new_form.app_id = existing_form.app_id
existing_form, new_form = apply_deprecation(existing_form, new_form)
return (existing_form, new_form)
def _get_form_json(self, xml_file, render_context=None):
# can't use .get_xml because we need to do encoding after string formatting
xml_string = self.get_file(xml_file, '.xml')
if render_context:
xml_string = str(xml_string).format(**render_context)
form_json = convert_xform_to_json(xml_string.encode('utf-8'))
return {
'form': form_json,
'domain': self.domain,
'xmlns': form_json['@xmlns'],
'doc_type': 'XFormInstance',
}
def should_ignore_submission(request):
"""
If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
the submissions should be ignored
"""
if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
return False
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
return False if from_demo_user(form_json) else True
def should_ignore_submission(request):
"""
If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
the submissions should be ignored
"""
if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
return False
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
return False if from_demo_user(form_json) else True
def _noauth_post(request, domain, app_id=None):
"""
This is explicitly called for a submission that has secure submissions enabled, but is manually
overriding the submit URL to not specify auth context. It appears to be used by demo mode.
It mainly just checks that we are touching test data only in the right domain and submitting
as demo_user.
"""
try:
instance, _ = couchforms.get_instance_and_attachment(request)
except BadSubmissionRequest as e:
return HttpResponseBadRequest(e.message)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
return (from_demo_user(form_json) or is_device_report(form_json))
def case_block_ok(case_updates):
"""
Check for all cases that we are submitting as demo_user and that the domain we
are submitting against for any previously existing cases matches the submission
domain.
"""
allowed_ids = ('demo_user', 'demo_user_group_id', None)
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in allowed_ids:
return False
if create_action.owner_id not in allowed_ids:
return False
if update_action:
if update_action.owner_id not in allowed_ids:
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
# todo: consider whether we want to remove this call, and/or pass the result
# through to the next function so we don't have to get the cases again later
cases = CommCareCase.objects.get_cases(list(case_ids), domain)
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id not in allowed_ids:
return False
def test_multiple_nodes(self):
form_data = convert_xform_to_json(self.get_xml('multiple_nodes'))
self.assertEqual(
_get_logs(form_data, 'log_subreport', 'log'),
[{'@date': '2016-03-20T20:46:08.664+05:30',
'msg': 'Logging out service login',
'type': 'maintenance'},
{'@date': '2016-03-20T20:46:08.988+05:30',
'msg': 'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8',
'type': 'user'},
{'@date': '2016-03-19T23:50:11.219+05:30',
'msg': 'Staging Sandbox: 7t3iyx01dxnn49a5xqt32916q5u7epn0',
'type': 'resources'}]
)
def test_multiple_nodes(self):
form_data = convert_xform_to_json(self.get_xml('multiple_nodes'))
self.assertEqual(
_get_logs(form_data, 'log_subreport', 'log'),
[{u'@date': u'2016-03-20T20:46:08.664+05:30',
u'msg': u'Logging out service login',
u'type': u'maintenance'},
{u'@date': u'2016-03-20T20:46:08.988+05:30',
u'msg': u'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8',
u'type': u'user'},
{u'@date': u'2016-03-19T23:50:11.219+05:30',
u'msg': u'Staging Sandbox: 7t3iyx01dxnn49a5xqt32916q5u7epn0',
u'type': u'resources'}]
)
def _create_new_xform(domain,
instance_xml,
attachments=None,
auth_context=None):
"""
create but do not save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
:param domain:
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
interface = FormProcessorInterface(domain)
assert attachments is not None
form_data = convert_xform_to_json(instance_xml)
if not form_data.get('@xmlns'):
raise MissingXMLNSError("Form is missing a required field: XMLNS")
adjust_datetimes(form_data)
xform = interface.new_xform(form_data)
xform.domain = domain
xform.auth_context = auth_context
# Maps all attachments to uniform format and adds form.xml to list before storing
attachments = map(
lambda a: Attachment(
name=a[0], raw_content=a[1], content_type=a[1].content_type),
attachments.items())
attachments.append(
Attachment(name='form.xml',
raw_content=instance_xml,
content_type='text/xml'))
interface.store_attachments(xform, attachments)
result = LockedFormProcessingResult(xform)
with ReleaseOnError(result.lock):
if interface.is_duplicate(xform.form_id):
raise DuplicateError(xform)
return result
def new_form_from_old(cls, existing_form, xml, value_responses_map, user_id):
from corehq.form_processor.parsers.form import apply_deprecation
new_xml = etree.tostring(xml)
form_json = convert_xform_to_json(new_xml)
new_form = cls.new_xform(form_json)
new_form.user_id = user_id
new_form.domain = existing_form.domain
new_form.app_id = existing_form.app_id
cls.store_attachments(new_form, [Attachment(
name=ATTACHMENT_NAME,
raw_content=new_xml,
content_type='text/xml',
)])
existing_form, new_form = apply_deprecation(existing_form, new_form)
return (existing_form, new_form)
def new_form_from_old(cls, existing_form, xml, value_responses_map, user_id):
from corehq.form_processor.parsers.form import apply_deprecation
new_xml = etree.tostring(xml)
form_json = convert_xform_to_json(new_xml)
new_form = cls.new_xform(form_json)
new_form.user_id = user_id
new_form.domain = existing_form.domain
new_form.app_id = existing_form.app_id
cls.store_attachments(new_form, [Attachment(
name=ATTACHMENT_NAME,
raw_content=new_xml,
content_type='text/xml',
)])
existing_form, new_form = apply_deprecation(existing_form, new_form)
return (existing_form, new_form)
def get_form_ready_to_save(metadata, is_db_test=False):
from corehq.form_processor.parsers.form import process_xform_xml
from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json
from corehq.form_processor.interfaces.processor import FormProcessorInterface
assert metadata is not None
metadata.domain = metadata.domain or uuid.uuid4().hex
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata)
if is_db_test:
wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form
else:
form_json = convert_xform_to_json(form_xml)
wrapped_form = FormProcessorInterface(domain=metadata.domain).new_xform(form_json)
wrapped_form.domain = metadata.domain
wrapped_form.received_on = metadata.received_on
return wrapped_form
def test_single_entry(self):
form_data = convert_xform_to_json(self.get_xml('single_entry'))
self.assertEqual(
_get_logs(form_data, 'user_error_subreport', 'user_error'),
[{"session": "frame: (COMMAND_ID m1)",
"user_id": "65t2l8ga654k93z92j236e2h30jt048b",
"expr": "",
"app_id": "skj94l95tw0k6v8esdj9s2g4chfpup83",
"version": "89",
"msg": ("XPath evaluation: type mismatch It looks like this "
"question contains a reference to path number which "
"evaluated to instance(item-list:numbers)/numbers_list"
"/numbers[1]/number which was not found. This often "
"means you forgot to include the full path to the "
"question -- e.g. /data/[node]"),
"@date": "2016-03-23T12:27:33.681-04",
"type": "error-config"}]
)
def test_pnc_form(self):
config, _ = get_datasource_config(self.ucr_name, 'icds-cas')
config.configured_indicators = [
ind for ind in config.configured_indicators
if ind['column_id'] != 'state_id'
]
form_json = convert_xform_to_json(self.get_xml('pnc_form_v10326'))
form_json = {
'form': form_json,
'domain': self.domain,
'xmlns': form_json['@xmlns'],
'doc_type': 'XFormInstance',
}
ucr_result = config.get_all_values(form_json)
for row in ucr_result:
row = {
i.column.database_column_name: i.value
for i in row
if i.column.database_column_name not in BLACKLISTED_COLUMNS
}
self.assertEqual(
{
"doc_id": None,
"repeat_iteration": 0,
"timeend": None,
"ccs_record_case_id":
"081cc405-5598-430f-ac8f-39cc4a1fdb30",
"child_health_case_id":
"252d8e20-c698-4c94-a5a9-53bbf8972b64",
"counsel_adequate_bf": None,
"counsel_breast": None,
"counsel_exclusive_bf": 1,
"counsel_increase_food_bf": 1,
"counsel_methods": None,
"counsel_only_milk": None,
"skin_to_skin": None,
"is_ebf": 1,
"water_or_milk": None,
"other_milk_to_child": 0,
"tea_other": None,
"eating": None,
"not_breastfeeding": None
}, row)
def test_ebf_form(self):
config, _ = get_datasource_config(self.ucr_name, 'icds-cas')
config.configured_indicators = [
ind for ind in config.configured_indicators
if ind['column_id'] != 'state_id'
]
form_json = convert_xform_to_json(self.get_xml('ebf_form_v10326'))
form_json = {
'form': form_json,
'domain': self.domain,
'xmlns': form_json['@xmlns'],
'doc_type': 'XFormInstance',
}
ucr_result = config.get_all_values(form_json)
for row in ucr_result:
row = {
i.column.database_column_name: i.value
for i in row
if i.column.database_column_name not in BLACKLISTED_COLUMNS
}
self.assertEqual(
{
"doc_id": None,
"repeat_iteration": 0,
"timeend": None,
"ccs_record_case_id":
"d53c940c-3bf3-44f7-97a1-f43fcbe74359",
"child_health_case_id":
"03f39da4-8ea3-4108-b8a8-1b58fdb4a698",
"counsel_adequate_bf": None,
"counsel_breast": None,
"counsel_exclusive_bf": None,
"counsel_increase_food_bf": None,
"counsel_methods": None,
"counsel_only_milk": 1,
"skin_to_skin": None,
"is_ebf": 1,
"water_or_milk": 0,
"other_milk_to_child": None,
"tea_other": 0,
"eating": 0,
"not_breastfeeding": None
}, row)
def convert_form_to_xml(form_data):
"""Convert form data dict to XML string
Does the inverse of `corehq.form_processor.utils.convert_xform_to_json()`
See also: submodules/xml2json/xml2json/lib.py
:param form_data: Form data dict: `<XFormInstance>.form_data`
:returns: XML string
:raises: ValueError if the input does not conform to expected
conventions.
:raises: RoundtripError if `convert_xform_to_json(xml)` produces a
different result that than the given `form_data`.
"""
tag = form_data.get("#type", "data")
element = convert_json_to_xml(form_data, tag)
xml = XML_PREFIX + tostring(element, pretty_print=True, encoding="unicode")
if convert_xform_to_json(xml) != form_data:
raise RoundtripError("to_json(to_xml(form_data)) != form_data")
return xml
def _create_new_xform(domain, instance_xml, attachments=None):
"""
create but do not save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
:param domain:
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
interface = FormProcessorInterface(domain)
assert attachments is not None
form_data = convert_xform_to_json(instance_xml)
if not form_data.get('@xmlns'):
raise MissingXMLNSError("Form is missing a required field: XMLNS")
adjust_datetimes(form_data)
xform = interface.new_xform(form_data)
xform.domain = domain
# Maps all attachments to uniform format and adds form.xml to list before storing
attachments = map(
lambda a: Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type),
attachments.items()
)
attachments.append(Attachment(name='form.xml', raw_content=instance_xml, content_type='text/xml'))
interface.store_attachments(xform, attachments)
result = LockedFormProcessingResult(xform)
with ReleaseOnError(result.lock):
if interface.is_duplicate(xform.form_id):
raise DuplicateError(xform)
return result
def get_form_ready_to_save(metadata, is_db_test=False):
from corehq.form_processor.parsers.form import process_xform_xml
from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.models import Attachment
assert metadata is not None
metadata.domain = metadata.domain or uuid.uuid4().hex
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata)
if is_db_test:
wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form
else:
interface = FormProcessorInterface(domain=metadata.domain)
form_json = convert_xform_to_json(form_xml)
wrapped_form = interface.new_xform(form_json)
wrapped_form.domain = metadata.domain
interface.store_attachments(wrapped_form, [
Attachment(name='form.xml', raw_content=form_xml, content_type='text/xml')
])
wrapped_form.received_on = metadata.received_on
wrapped_form.app_id = metadata.app_id
return wrapped_form
def _noauth_post(request, domain, app_id=None):
"""
This is explictly called for a submission that has secure submissions enabled, but is manually
overriding the submit URL to not specify auth context. It appears to be used by demo mode.
It mainly just checks that we are touching test data only in the right domain and submitting
as demo_user.
"""
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
try:
# require new-style meta/userID (reject Meta/chw_id)
if form_json['meta']['userID'] == 'demo_user':
return True
except (KeyError, ValueError):
pass
if is_device_report(form_json):
return True
return False
def case_block_ok(case_updates):
"""
Check for all cases that we are submitting as demo_user and that the domain we
are submitting against for any previously existing cases matches the submission
domain.
"""
allowed_ids = ('demo_user', 'demo_user_group_id', None)
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in allowed_ids:
return False
if create_action.owner_id not in allowed_ids:
return False
if update_action:
if update_action.owner_id not in allowed_ids:
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
# todo: consider whether we want to remove this call, and/or pass the result
# through to the next function so we don't have to get the cases again later
cases = CommCareCase.bulk_get_lite(list(case_ids))
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id not in allowed_ids:
return False
return True
if not (form_ok(form_json) and case_block_ok(case_updates)):
return HttpResponseForbidden()
return _process_form(
request=request,
domain=domain,
app_id=app_id,
user_id=None,
authenticated=False,
auth_cls=WaivedAuthContext,
)
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
rate_limit_submission_by_delaying(domain, max_wait=15)
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request,
"Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=bool(request.GET.get('force_logs', False)),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request,
"XFormLockError: %s" % err,
XFORM_LOCKED_COUNT,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _get_xform(self, filename):
xform = FormProcessorInterface(self.domain).new_xform(
convert_xform_to_json(self.get_xml(filename)))
xform.received_on = self.received_on
return xform
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {
'backend': 'sql' if should_use_sql_backend(domain) else 'couch',
'domain': domain
}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
if isinstance(instance, BadRequest):
response = HttpResponseBadRequest(instance.message)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
)
try:
result = submission_post.run()
except XFormLockError as err:
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(request, "Received a submission with POST.keys()",
details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
with TimingContext() as timer:
result = submission_post.run()
response = result.response
tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
u'domain:{}'.format(domain)
]
datadog_counter('commcare.xform_submissions.count',
tags=tags +
['status_code:{}'.format(response.status_code)])
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
elif response.status_code == 201:
datadog_gauge('commcare.xform_submissions.timings',
timer.duration,
tags=tags)
# normalize over number of items (form or case) saved
normalized_time = timer.duration / (1 + len(result.cases))
datadog_gauge('commcare.xform_submissions.normalized_timings',
normalized_time,
tags=tags)
datadog_counter('commcare.xform_submissions.case_count',
len(result.cases),
tags=tags)
datadog_counter('commcare.xform_submissions.ledger_count',
len(result.ledgers),
tags=tags)
return response
def json_to_xml_to_json(form_json):
form_data = json.loads(form_json)
xml = convert_form_to_xml(form_data)
print(xml)
eq(convert_xform_to_json(xml), form_data)
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request, "Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags,
domain, app_id, user_id, authenticated, meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT,
metric_tags, domain, app_id, user_id, authenticated, status=423,
notify=False,
)
response = result.response
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
_record_metrics(metric_tags, result.submission_type, response, result, timer)
return response
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(
request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = [
"domain:{}".format(domain),
"app_id:{}".format(app_id),
"user_id:{}".format(user_id),
"authenticated:{}".format(authenticated),
"form_meta:{}".format(meta),
]
datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details)
notify_exception(request, "Received a submission with POST.keys()",
details)
response = HttpResponseBadRequest(six.text_type(e))
_record_metrics(metric_tags, 'unknown', response)
return response
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
result = submission_post.run()
response = result.response
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
_record_metrics(metric_tags, result.submission_type, response, result,
timer)
return response
def _get_xform(self, filename):
xform = FormProcessorInterface(self.domain).new_xform(convert_xform_to_json(self.get_xml(filename)))
xform.received_on = self.received_on
return xform
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if rate_limit_submission(domain):
return HttpTooManyRequests()
metric_tags = {'backend': 'sql', 'domain': domain}
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except Exception:
meta = {}
metrics_counter('commcare.corrupt_multimedia_submissions',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"Received a submission with POST.keys()",
metric_tags,
domain,
app_id,
user_id,
authenticated,
meta,
)
# the order of these exceptions is relevant
except UnprocessableFormSubmission as e:
return openrosa_response.OpenRosaResponse(
message=e.message,
nature=openrosa_response.ResponseNature.PROCESSING_FAILURE,
status=e.status_code,
).response()
except BadSubmissionRequest as e:
response = HttpResponse(e.message, status=e.status_code)
_record_metrics(metric_tags, 'known_failures', response)
return response
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
force_logs=request.GET.get('force_logs', 'false') == 'true',
timing_context=timer)
try:
result = submission_post.run()
except XFormLockError as err:
logging.warning('Unable to get lock for form %s', err)
metrics_counter('commcare.xformlocked.count',
tags={
'domain': domain,
'authenticated': authenticated
})
return _submission_error(
request,
"XFormLockError: %s" % err,
metric_tags,
domain,
app_id,
user_id,
authenticated,
status=423,
notify=False,
)
response = result.response
response.request_timer = timer # logged as Sentry breadcrumbs in LogLongRequestMiddleware
_record_metrics(metric_tags, result.submission_type, result.response,
timer, result.xform)
return response
def _noauth_post(request, domain, app_id=None):
"""
This is explictly called for a submission that has secure submissions enabled, but is manually
overriding the submit URL to not specify auth context. It appears to be used by demo mode.
It mainly just checks that we are touching test data only in the right domain and submitting
as demo_user.
"""
instance, _ = couchforms.get_instance_and_attachment(request)
form_json = convert_xform_to_json(instance)
case_updates = get_case_updates(form_json)
def form_ok(form_json):
return (from_demo_user(form_json) or is_device_report(form_json))
def case_block_ok(case_updates):
"""
Check for all cases that we are submitting as demo_user and that the domain we
are submitting against for any previously existing cases matches the submission
domain.
"""
allowed_ids = ('demo_user', 'demo_user_group_id', None)
case_ids = set()
for case_update in case_updates:
case_ids.add(case_update.id)
create_action = case_update.get_create_action()
update_action = case_update.get_update_action()
index_action = case_update.get_index_action()
if create_action:
if create_action.user_id not in allowed_ids:
return False
if create_action.owner_id not in allowed_ids:
return False
if update_action:
if update_action.owner_id not in allowed_ids:
return False
if index_action:
for index in index_action.indices:
case_ids.add(index.referenced_id)
# todo: consider whether we want to remove this call, and/or pass the result
# through to the next function so we don't have to get the cases again later
cases = CaseAccessors(domain).get_cases(list(case_ids))
for case in cases:
if case.domain != domain:
return False
if case.owner_id or case.user_id not in allowed_ids:
return False
return True
if not (form_ok(form_json) and case_block_ok(case_updates)):
if request.GET.get('submit_mode') != DEMO_SUBMIT_MODE:
# invalid submissions under demo mode submission can be processed
return HttpResponseForbidden()
return _process_form(
request=request,
domain=domain,
app_id=app_id,
user_id=None,
authenticated=False,
auth_cls=WaivedAuthContext,
)
