def should_ignore_submission(request): """ If IGNORE_ALL_DEMO_USER_SUBMISSIONS is True then ignore submission if from demo user. Else If submission request.GET has `submit_mode=demo` and submitting user is not demo_user, the submissions should be ignored """ form_json = None if IGNORE_ALL_DEMO_USER_SUBMISSIONS: instance, _ = couchforms.get_instance_and_attachment(request) try: form_json = convert_xform_to_json(instance) except couchforms.XMLSyntaxError: # let the usual workflow handle response for invalid xml return False else: form_meta = form_json.get('meta') if form_meta and _submitted_by_demo_user(form_meta, request.domain): _notify_submission_if_applicable(request, form_meta) return True if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE: return False if form_json is None: instance, _ = couchforms.get_instance_and_attachment(request) form_json = convert_xform_to_json(instance) return False if from_demo_user(form_json) else True
def get_form_ready_to_save(metadata, is_db_test=False): from corehq.form_processor.parsers.form import process_xform_xml from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.models import Attachment assert metadata is not None metadata.domain = metadata.domain or uuid.uuid4().hex form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata) if is_db_test: wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form else: interface = FormProcessorInterface(domain=metadata.domain) form_json = convert_xform_to_json(form_xml) wrapped_form = interface.new_xform(form_json) wrapped_form.domain = metadata.domain interface.store_attachments(wrapped_form, [ Attachment( name='form.xml', raw_content=form_xml, content_type='text/xml') ]) wrapped_form.received_on = metadata.received_on wrapped_form.app_id = metadata.app_id return wrapped_form
def _get_new_form_json(xml, xform_id): form_json = convert_xform_to_json(xml) with force_phone_timezones_should_be_processed(): adjust_datetimes(form_json) # this is actually in-place because of how jsonobject works scrub_meta(XFormInstance.wrap({'form': form_json, '_id': xform_id})) return form_json
def test_single_entry(self): form_data = convert_xform_to_json(self.get_xml('single_entry')) self.assertEqual( _get_logs(form_data, 'user_error_subreport', 'user_error'), [{ "session": "frame: (COMMAND_ID m1)", "user_id": "65t2l8ga654k93z92j236e2h30jt048b", "expr": "", "app_id": "skj94l95tw0k6v8esdj9s2g4chfpup83", "version": "89", "msg": ("XPath evaluation: type mismatch It looks like this " "question contains a reference to path number which " "evaluated to instance(item-list:numbers)/numbers_list" "/numbers[1]/number which was not found. This often " "means you forgot to include the full path to the " "question -- e.g. /data/[node]"), "@date": "2016-03-23T12:27:33.681-04", "type": "error-config" }])
def create_case(case_id, files, patient_case_id=None): """ Handle case submission for the sonosite endpoint """ # we already parsed what we need from this, so can just remove it # without worrying we will need it later files.pop('PT_PPS.XML', '') xform = render_sonosite_xform(files, case_id, patient_case_id) file_dict = {} for f in files: file_dict[f] = UploadedFile(files[f], f) submit_form_locally( instance=xform, attachments=file_dict, domain=UTH_DOMAIN, ) # this is a bit of a hack / abstraction violation # would be nice if submit_form_locally returned info about cases updated case_ids = { case_update.id for case_update in get_case_updates(convert_xform_to_json(xform)) } return [CommCareCase.get(case_id) for case_id in case_ids]
def _get_form_json(self, xml_file): form_json = convert_xform_to_json(self.get_xml(xml_file)) return { 'form': form_json, 'domain': self.domain, 'xmlns': form_json['@xmlns'], 'doc_type': 'XFormInstance', }
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} details = { "domain": domain, "app_id": app_id, "user_id": user_id, "authenticated": authenticated, "form_meta": meta, } log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details) notify_exception(None, "Received a submission with POST.keys()", details) return HttpResponseBadRequest(e.message) app_id, build_id = get_app_and_build_ids(domain, app_id) response = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), ).get_response() if response.status_code == 400: db_response = get_db('couchlog').save_doc({ 'request': unicode(request), 'response': unicode(response), }) logging.error( 'Status code 400 for a form submission. ' 'Response is: \n{0}\n' 'See couchlog db for more info: {1}'.format( unicode(response), db_response['id'], ) ) return response
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria return SubmissionPost.submission_ignored_response() if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): return SubmissionPost.get_blacklisted_response() try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} details = { "domain": domain, "app_id": app_id, "user_id": user_id, "authenticated": authenticated, "form_meta": meta, } log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details) notify_exception(None, "Received a submission with POST.keys()", details) return HttpResponseBadRequest(e.message) app_id, build_id = get_app_and_build_ids(domain, app_id) response = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), ).get_response() if response.status_code == 400: logging.error( 'Status code 400 for a form submission. ' 'Response is: \n{0}\n' ) return response
def test_single_node(self): form_data = convert_xform_to_json(self.get_xml('single_node')) self.assertEqual( _get_logs(form_data, 'log_subreport', 'log'), [{'@date': '2016-03-20T20:46:08.664+05:30', 'msg': 'Logging out service login', 'type': 'maintenance'}, {'@date': '2016-03-20T20:46:08.988+05:30', 'msg': 'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8', 'type': 'user'}] )
def test_single_node(self): form_data = convert_xform_to_json(self.get_xml('single_node')) self.assertEqual( _get_logs(form_data, 'log_subreport', 'log'), [{u'@date': u'2016-03-20T20:46:08.664+05:30', u'msg': u'Logging out service login', u'type': u'maintenance'}, {u'@date': u'2016-03-20T20:46:08.988+05:30', u'msg': u'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8', u'type': u'user'}] )
def new_form_from_old(cls, existing_form, xml, value_responses_map, user_id): from corehq.form_processor.parsers.form import apply_deprecation new_xml = etree.tostring(xml) form_json = convert_xform_to_json(new_xml) new_form = cls.new_xform(form_json) new_form.user_id = user_id new_form.domain = existing_form.domain new_form.app_id = existing_form.app_id existing_form, new_form = apply_deprecation(existing_form, new_form) return (existing_form, new_form)
def _get_form_json(self, xml_file, render_context=None): # can't use .get_xml because we need to do encoding after string formatting xml_string = self.get_file(xml_file, '.xml') if render_context: xml_string = str(xml_string).format(**render_context) form_json = convert_xform_to_json(xml_string.encode('utf-8')) return { 'form': form_json, 'domain': self.domain, 'xmlns': form_json['@xmlns'], 'doc_type': 'XFormInstance', }
def should_ignore_submission(request): """ If submission request.GET has `submit_mode=demo` and submitting user is not demo_user, the submissions should be ignored """ if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE: return False instance, _ = couchforms.get_instance_and_attachment(request) form_json = convert_xform_to_json(instance) return False if from_demo_user(form_json) else True
def _noauth_post(request, domain, app_id=None): """ This is explicitly called for a submission that has secure submissions enabled, but is manually overriding the submit URL to not specify auth context. It appears to be used by demo mode. It mainly just checks that we are touching test data only in the right domain and submitting as demo_user. """ try: instance, _ = couchforms.get_instance_and_attachment(request) except BadSubmissionRequest as e: return HttpResponseBadRequest(e.message) form_json = convert_xform_to_json(instance) case_updates = get_case_updates(form_json) def form_ok(form_json): return (from_demo_user(form_json) or is_device_report(form_json)) def case_block_ok(case_updates): """ Check for all cases that we are submitting as demo_user and that the domain we are submitting against for any previously existing cases matches the submission domain. """ allowed_ids = ('demo_user', 'demo_user_group_id', None) case_ids = set() for case_update in case_updates: case_ids.add(case_update.id) create_action = case_update.get_create_action() update_action = case_update.get_update_action() index_action = case_update.get_index_action() if create_action: if create_action.user_id not in allowed_ids: return False if create_action.owner_id not in allowed_ids: return False if update_action: if update_action.owner_id not in allowed_ids: return False if index_action: for index in index_action.indices: case_ids.add(index.referenced_id) # todo: consider whether we want to remove this call, and/or pass the result # through to the next function so we don't have to get the cases again later cases = CommCareCase.objects.get_cases(list(case_ids), domain) for case in cases: if case.domain != domain: return False if case.owner_id or case.user_id not in allowed_ids: return False
def test_multiple_nodes(self): form_data = convert_xform_to_json(self.get_xml('multiple_nodes')) self.assertEqual( _get_logs(form_data, 'log_subreport', 'log'), [{'@date': '2016-03-20T20:46:08.664+05:30', 'msg': 'Logging out service login', 'type': 'maintenance'}, {'@date': '2016-03-20T20:46:08.988+05:30', 'msg': 'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8', 'type': 'user'}, {'@date': '2016-03-19T23:50:11.219+05:30', 'msg': 'Staging Sandbox: 7t3iyx01dxnn49a5xqt32916q5u7epn0', 'type': 'resources'}] )
def test_multiple_nodes(self): form_data = convert_xform_to_json(self.get_xml('multiple_nodes')) self.assertEqual( _get_logs(form_data, 'log_subreport', 'log'), [{u'@date': u'2016-03-20T20:46:08.664+05:30', u'msg': u'Logging out service login', u'type': u'maintenance'}, {u'@date': u'2016-03-20T20:46:08.988+05:30', u'msg': u'login|user.test|gxpg40k9lh9w7853w3gc91o1g7zu1wi8', u'type': u'user'}, {u'@date': u'2016-03-19T23:50:11.219+05:30', u'msg': u'Staging Sandbox: 7t3iyx01dxnn49a5xqt32916q5u7epn0', u'type': u'resources'}] )
def _create_new_xform(domain, instance_xml, attachments=None, auth_context=None): """ create but do not save an XFormInstance from an xform payload (xml_string) optionally set the doc _id to a predefined value (_id) return doc _id of the created doc `process` is transformation to apply to the form right before saving This is to avoid having to save multiple times If xml_string is bad xml - raise couchforms.XMLSyntaxError :param domain: """ from corehq.form_processor.interfaces.processor import FormProcessorInterface interface = FormProcessorInterface(domain) assert attachments is not None form_data = convert_xform_to_json(instance_xml) if not form_data.get('@xmlns'): raise MissingXMLNSError("Form is missing a required field: XMLNS") adjust_datetimes(form_data) xform = interface.new_xform(form_data) xform.domain = domain xform.auth_context = auth_context # Maps all attachments to uniform format and adds form.xml to list before storing attachments = map( lambda a: Attachment( name=a[0], raw_content=a[1], content_type=a[1].content_type), attachments.items()) attachments.append( Attachment(name='form.xml', raw_content=instance_xml, content_type='text/xml')) interface.store_attachments(xform, attachments) result = LockedFormProcessingResult(xform) with ReleaseOnError(result.lock): if interface.is_duplicate(xform.form_id): raise DuplicateError(xform) return result
def new_form_from_old(cls, existing_form, xml, value_responses_map, user_id): from corehq.form_processor.parsers.form import apply_deprecation new_xml = etree.tostring(xml) form_json = convert_xform_to_json(new_xml) new_form = cls.new_xform(form_json) new_form.user_id = user_id new_form.domain = existing_form.domain new_form.app_id = existing_form.app_id cls.store_attachments(new_form, [Attachment( name=ATTACHMENT_NAME, raw_content=new_xml, content_type='text/xml', )]) existing_form, new_form = apply_deprecation(existing_form, new_form) return (existing_form, new_form)
def get_form_ready_to_save(metadata, is_db_test=False): from corehq.form_processor.parsers.form import process_xform_xml from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json from corehq.form_processor.interfaces.processor import FormProcessorInterface assert metadata is not None metadata.domain = metadata.domain or uuid.uuid4().hex form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata) if is_db_test: wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form else: form_json = convert_xform_to_json(form_xml) wrapped_form = FormProcessorInterface(domain=metadata.domain).new_xform(form_json) wrapped_form.domain = metadata.domain wrapped_form.received_on = metadata.received_on return wrapped_form
def test_single_entry(self): form_data = convert_xform_to_json(self.get_xml('single_entry')) self.assertEqual( _get_logs(form_data, 'user_error_subreport', 'user_error'), [{"session": "frame: (COMMAND_ID m1)", "user_id": "65t2l8ga654k93z92j236e2h30jt048b", "expr": "", "app_id": "skj94l95tw0k6v8esdj9s2g4chfpup83", "version": "89", "msg": ("XPath evaluation: type mismatch It looks like this " "question contains a reference to path number which " "evaluated to instance(item-list:numbers)/numbers_list" "/numbers[1]/number which was not found. This often " "means you forgot to include the full path to the " "question -- e.g. /data/[node]"), "@date": "2016-03-23T12:27:33.681-04", "type": "error-config"}] )
def test_pnc_form(self): config, _ = get_datasource_config(self.ucr_name, 'icds-cas') config.configured_indicators = [ ind for ind in config.configured_indicators if ind['column_id'] != 'state_id' ] form_json = convert_xform_to_json(self.get_xml('pnc_form_v10326')) form_json = { 'form': form_json, 'domain': self.domain, 'xmlns': form_json['@xmlns'], 'doc_type': 'XFormInstance', } ucr_result = config.get_all_values(form_json) for row in ucr_result: row = { i.column.database_column_name: i.value for i in row if i.column.database_column_name not in BLACKLISTED_COLUMNS } self.assertEqual( { "doc_id": None, "repeat_iteration": 0, "timeend": None, "ccs_record_case_id": "081cc405-5598-430f-ac8f-39cc4a1fdb30", "child_health_case_id": "252d8e20-c698-4c94-a5a9-53bbf8972b64", "counsel_adequate_bf": None, "counsel_breast": None, "counsel_exclusive_bf": 1, "counsel_increase_food_bf": 1, "counsel_methods": None, "counsel_only_milk": None, "skin_to_skin": None, "is_ebf": 1, "water_or_milk": None, "other_milk_to_child": 0, "tea_other": None, "eating": None, "not_breastfeeding": None }, row)
def test_ebf_form(self): config, _ = get_datasource_config(self.ucr_name, 'icds-cas') config.configured_indicators = [ ind for ind in config.configured_indicators if ind['column_id'] != 'state_id' ] form_json = convert_xform_to_json(self.get_xml('ebf_form_v10326')) form_json = { 'form': form_json, 'domain': self.domain, 'xmlns': form_json['@xmlns'], 'doc_type': 'XFormInstance', } ucr_result = config.get_all_values(form_json) for row in ucr_result: row = { i.column.database_column_name: i.value for i in row if i.column.database_column_name not in BLACKLISTED_COLUMNS } self.assertEqual( { "doc_id": None, "repeat_iteration": 0, "timeend": None, "ccs_record_case_id": "d53c940c-3bf3-44f7-97a1-f43fcbe74359", "child_health_case_id": "03f39da4-8ea3-4108-b8a8-1b58fdb4a698", "counsel_adequate_bf": None, "counsel_breast": None, "counsel_exclusive_bf": None, "counsel_increase_food_bf": None, "counsel_methods": None, "counsel_only_milk": 1, "skin_to_skin": None, "is_ebf": 1, "water_or_milk": 0, "other_milk_to_child": None, "tea_other": 0, "eating": 0, "not_breastfeeding": None }, row)
def convert_form_to_xml(form_data): """Convert form data dict to XML string Does the inverse of `corehq.form_processor.utils.convert_xform_to_json()` See also: submodules/xml2json/xml2json/lib.py :param form_data: Form data dict: `<XFormInstance>.form_data` :returns: XML string :raises: ValueError if the input does not conform to expected conventions. :raises: RoundtripError if `convert_xform_to_json(xml)` produces a different result that than the given `form_data`. """ tag = form_data.get("#type", "data") element = convert_json_to_xml(form_data, tag) xml = XML_PREFIX + tostring(element, pretty_print=True, encoding="unicode") if convert_xform_to_json(xml) != form_data: raise RoundtripError("to_json(to_xml(form_data)) != form_data") return xml
def _create_new_xform(domain, instance_xml, attachments=None): """ create but do not save an XFormInstance from an xform payload (xml_string) optionally set the doc _id to a predefined value (_id) return doc _id of the created doc `process` is transformation to apply to the form right before saving This is to avoid having to save multiple times If xml_string is bad xml - raise couchforms.XMLSyntaxError :param domain: """ from corehq.form_processor.interfaces.processor import FormProcessorInterface interface = FormProcessorInterface(domain) assert attachments is not None form_data = convert_xform_to_json(instance_xml) if not form_data.get('@xmlns'): raise MissingXMLNSError("Form is missing a required field: XMLNS") adjust_datetimes(form_data) xform = interface.new_xform(form_data) xform.domain = domain # Maps all attachments to uniform format and adds form.xml to list before storing attachments = map( lambda a: Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type), attachments.items() ) attachments.append(Attachment(name='form.xml', raw_content=instance_xml, content_type='text/xml')) interface.store_attachments(xform, attachments) result = LockedFormProcessingResult(xform) with ReleaseOnError(result.lock): if interface.is_duplicate(xform.form_id): raise DuplicateError(xform) return result
def get_form_ready_to_save(metadata, is_db_test=False): from corehq.form_processor.parsers.form import process_xform_xml from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.models import Attachment assert metadata is not None metadata.domain = metadata.domain or uuid.uuid4().hex form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata) if is_db_test: wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form else: interface = FormProcessorInterface(domain=metadata.domain) form_json = convert_xform_to_json(form_xml) wrapped_form = interface.new_xform(form_json) wrapped_form.domain = metadata.domain interface.store_attachments(wrapped_form, [ Attachment(name='form.xml', raw_content=form_xml, content_type='text/xml') ]) wrapped_form.received_on = metadata.received_on wrapped_form.app_id = metadata.app_id return wrapped_form
def _noauth_post(request, domain, app_id=None): """ This is explictly called for a submission that has secure submissions enabled, but is manually overriding the submit URL to not specify auth context. It appears to be used by demo mode. It mainly just checks that we are touching test data only in the right domain and submitting as demo_user. """ instance, _ = couchforms.get_instance_and_attachment(request) form_json = convert_xform_to_json(instance) case_updates = get_case_updates(form_json) def form_ok(form_json): try: # require new-style meta/userID (reject Meta/chw_id) if form_json['meta']['userID'] == 'demo_user': return True except (KeyError, ValueError): pass if is_device_report(form_json): return True return False def case_block_ok(case_updates): """ Check for all cases that we are submitting as demo_user and that the domain we are submitting against for any previously existing cases matches the submission domain. """ allowed_ids = ('demo_user', 'demo_user_group_id', None) case_ids = set() for case_update in case_updates: case_ids.add(case_update.id) create_action = case_update.get_create_action() update_action = case_update.get_update_action() index_action = case_update.get_index_action() if create_action: if create_action.user_id not in allowed_ids: return False if create_action.owner_id not in allowed_ids: return False if update_action: if update_action.owner_id not in allowed_ids: return False if index_action: for index in index_action.indices: case_ids.add(index.referenced_id) # todo: consider whether we want to remove this call, and/or pass the result # through to the next function so we don't have to get the cases again later cases = CommCareCase.bulk_get_lite(list(case_ids)) for case in cases: if case.domain != domain: return False if case.owner_id or case.user_id not in allowed_ids: return False return True if not (form_ok(form_json) and case_block_ok(case_updates)): return HttpResponseForbidden() return _process_form( request=request, domain=domain, app_id=app_id, user_id=None, authenticated=False, auth_cls=WaivedAuthContext, )
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): rate_limit_submission_by_delaying(domain, max_wait=15) metric_tags = [ 'backend:sql' if should_use_sql_backend(domain) else 'backend:couch', 'domain:{}'.format(domain), ] if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria response = openrosa_response.SUBMISSION_IGNORED_RESPONSE _record_metrics(metric_tags, 'ignored', response) return response if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): response = openrosa_response.BLACKLISTED_RESPONSE _record_metrics(metric_tags, 'blacklisted', response) return response with TimingContext() as timer: try: instance, attachments = couchforms.get_instance_and_attachment( request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} return _submission_error( request, "Received a submission with POST.keys()", MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags, domain, app_id, user_id, authenticated, meta, ) app_id, build_id = get_app_and_build_ids(domain, app_id) submission_post = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), force_logs=bool(request.GET.get('force_logs', False)), ) try: result = submission_post.run() except XFormLockError as err: return _submission_error( request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT, metric_tags, domain, app_id, user_id, authenticated, status=423, notify=False, ) response = result.response _record_metrics(metric_tags, result.submission_type, result.response, timer, result.xform) return response
def _get_xform(self, filename): xform = FormProcessorInterface(self.domain).new_xform( convert_xform_to_json(self.get_xml(filename))) xform.received_on = self.received_on return xform
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): if rate_limit_submission(domain): return HttpTooManyRequests() metric_tags = { 'backend': 'sql' if should_use_sql_backend(domain) else 'couch', 'domain': domain } try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} metrics_counter('commcare.corrupt_multimedia_submissions', tags={ 'domain': domain, 'authenticated': authenticated }) return _submission_error( request, "Received a submission with POST.keys()", metric_tags, domain, app_id, user_id, authenticated, meta, ) if isinstance(instance, BadRequest): response = HttpResponseBadRequest(instance.message) _record_metrics(metric_tags, 'known_failures', response) return response if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria response = openrosa_response.SUBMISSION_IGNORED_RESPONSE _record_metrics(metric_tags, 'ignored', response) return response if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): response = openrosa_response.BLACKLISTED_RESPONSE _record_metrics(metric_tags, 'blacklisted', response) return response with TimingContext() as timer: app_id, build_id = get_app_and_build_ids(domain, app_id) submission_post = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), force_logs=request.GET.get('force_logs', 'false') == 'true', ) try: result = submission_post.run() except XFormLockError as err: metrics_counter('commcare.xformlocked.count', tags={ 'domain': domain, 'authenticated': authenticated }) return _submission_error( request, "XFormLockError: %s" % err, metric_tags, domain, app_id, user_id, authenticated, status=423, notify=False, ) response = result.response _record_metrics(metric_tags, result.submission_type, result.response, timer, result.xform) return response
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria return SubmissionPost.submission_ignored_response() try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} details = { "domain": domain, "app_id": app_id, "user_id": user_id, "authenticated": authenticated, "form_meta": meta, } log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details) notify_exception(None, "Received a submission with POST.keys()", details) return HttpResponseBadRequest(e.message) app_id, build_id = get_app_and_build_ids(domain, app_id) response = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), ).get_response() if response.status_code == 400: db_response = get_db('couchlog').save_doc({ 'request': unicode(request), 'response': unicode(response), }) logging.error( 'Status code 400 for a form submission. ' 'Response is: \n{0}\n' 'See couchlog db for more info: {1}'.format( unicode(response), db_response['id'], ) ) return response
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria return SubmissionPost.submission_ignored_response() if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): return SubmissionPost.get_blacklisted_response() try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} details = { "domain": domain, "app_id": app_id, "user_id": user_id, "authenticated": authenticated, "form_meta": meta, } log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details) notify_exception(request, "Received a submission with POST.keys()", details) return HttpResponseBadRequest(e.message) app_id, build_id = get_app_and_build_ids(domain, app_id) submission_post = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), ) with TimingContext() as timer: result = submission_post.run() response = result.response tags = [ 'backend:sql' if should_use_sql_backend(domain) else 'backend:couch', u'domain:{}'.format(domain) ] datadog_counter('commcare.xform_submissions.count', tags=tags + ['status_code:{}'.format(response.status_code)]) if response.status_code == 400: logging.error('Status code 400 for a form submission. ' 'Response is: \n{0}\n') elif response.status_code == 201: datadog_gauge('commcare.xform_submissions.timings', timer.duration, tags=tags) # normalize over number of items (form or case) saved normalized_time = timer.duration / (1 + len(result.cases)) datadog_gauge('commcare.xform_submissions.normalized_timings', normalized_time, tags=tags) datadog_counter('commcare.xform_submissions.case_count', len(result.cases), tags=tags) datadog_counter('commcare.xform_submissions.ledger_count', len(result.ledgers), tags=tags) return response
def json_to_xml_to_json(form_json): form_data = json.loads(form_json) xml = convert_form_to_xml(form_data) print(xml) eq(convert_xform_to_json(xml), form_data)
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): metric_tags = [ 'backend:sql' if should_use_sql_backend(domain) else 'backend:couch', 'domain:{}'.format(domain), ] if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria response = openrosa_response.SUBMISSION_IGNORED_RESPONSE _record_metrics(metric_tags, 'ignored', response) return response if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): response = openrosa_response.BLACKLISTED_RESPONSE _record_metrics(metric_tags, 'blacklisted', response) return response with TimingContext() as timer: try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} return _submission_error( request, "Received a submission with POST.keys()", MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags, domain, app_id, user_id, authenticated, meta, ) app_id, build_id = get_app_and_build_ids(domain, app_id) submission_post = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), ) try: result = submission_post.run() except XFormLockError as err: return _submission_error( request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT, metric_tags, domain, app_id, user_id, authenticated, status=423, notify=False, ) response = result.response if response.status_code == 400: logging.error( 'Status code 400 for a form submission. ' 'Response is: \n{0}\n' ) _record_metrics(metric_tags, result.submission_type, response, result, timer) return response
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): metric_tags = [ 'backend:sql' if should_use_sql_backend(domain) else 'backend:couch', 'domain:{}'.format(domain), ] if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria response = openrosa_response.SUBMISSION_IGNORED_RESPONSE _record_metrics(metric_tags, 'ignored', response) return response if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): response = openrosa_response.BLACKLISTED_RESPONSE _record_metrics(metric_tags, 'blacklisted', response) return response with TimingContext() as timer: try: instance, attachments = couchforms.get_instance_and_attachment( request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} details = [ "domain:{}".format(domain), "app_id:{}".format(app_id), "user_id:{}".format(user_id), "authenticated:{}".format(authenticated), "form_meta:{}".format(meta), ] datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details) notify_exception(request, "Received a submission with POST.keys()", details) response = HttpResponseBadRequest(six.text_type(e)) _record_metrics(metric_tags, 'unknown', response) return response app_id, build_id = get_app_and_build_ids(domain, app_id) submission_post = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), ) result = submission_post.run() response = result.response if response.status_code == 400: logging.error('Status code 400 for a form submission. ' 'Response is: \n{0}\n') _record_metrics(metric_tags, result.submission_type, response, result, timer) return response
def _get_xform(self, filename): xform = FormProcessorInterface(self.domain).new_xform(convert_xform_to_json(self.get_xml(filename))) xform.received_on = self.received_on return xform
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): if rate_limit_submission(domain): return HttpTooManyRequests() metric_tags = {'backend': 'sql', 'domain': domain} try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except Exception: meta = {} metrics_counter('commcare.corrupt_multimedia_submissions', tags={ 'domain': domain, 'authenticated': authenticated }) return _submission_error( request, "Received a submission with POST.keys()", metric_tags, domain, app_id, user_id, authenticated, meta, ) # the order of these exceptions is relevant except UnprocessableFormSubmission as e: return openrosa_response.OpenRosaResponse( message=e.message, nature=openrosa_response.ResponseNature.PROCESSING_FAILURE, status=e.status_code, ).response() except BadSubmissionRequest as e: response = HttpResponse(e.message, status=e.status_code) _record_metrics(metric_tags, 'known_failures', response) return response if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria response = openrosa_response.SUBMISSION_IGNORED_RESPONSE _record_metrics(metric_tags, 'ignored', response) return response if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): response = openrosa_response.BLACKLISTED_RESPONSE _record_metrics(metric_tags, 'blacklisted', response) return response with TimingContext() as timer: app_id, build_id = get_app_and_build_ids(domain, app_id) submission_post = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), force_logs=request.GET.get('force_logs', 'false') == 'true', timing_context=timer) try: result = submission_post.run() except XFormLockError as err: logging.warning('Unable to get lock for form %s', err) metrics_counter('commcare.xformlocked.count', tags={ 'domain': domain, 'authenticated': authenticated }) return _submission_error( request, "XFormLockError: %s" % err, metric_tags, domain, app_id, user_id, authenticated, status=423, notify=False, ) response = result.response response.request_timer = timer # logged as Sentry breadcrumbs in LogLongRequestMiddleware _record_metrics(metric_tags, result.submission_type, result.response, timer, result.xform) return response
def _noauth_post(request, domain, app_id=None): """ This is explictly called for a submission that has secure submissions enabled, but is manually overriding the submit URL to not specify auth context. It appears to be used by demo mode. It mainly just checks that we are touching test data only in the right domain and submitting as demo_user. """ instance, _ = couchforms.get_instance_and_attachment(request) form_json = convert_xform_to_json(instance) case_updates = get_case_updates(form_json) def form_ok(form_json): return (from_demo_user(form_json) or is_device_report(form_json)) def case_block_ok(case_updates): """ Check for all cases that we are submitting as demo_user and that the domain we are submitting against for any previously existing cases matches the submission domain. """ allowed_ids = ('demo_user', 'demo_user_group_id', None) case_ids = set() for case_update in case_updates: case_ids.add(case_update.id) create_action = case_update.get_create_action() update_action = case_update.get_update_action() index_action = case_update.get_index_action() if create_action: if create_action.user_id not in allowed_ids: return False if create_action.owner_id not in allowed_ids: return False if update_action: if update_action.owner_id not in allowed_ids: return False if index_action: for index in index_action.indices: case_ids.add(index.referenced_id) # todo: consider whether we want to remove this call, and/or pass the result # through to the next function so we don't have to get the cases again later cases = CaseAccessors(domain).get_cases(list(case_ids)) for case in cases: if case.domain != domain: return False if case.owner_id or case.user_id not in allowed_ids: return False return True if not (form_ok(form_json) and case_block_ok(case_updates)): if request.GET.get('submit_mode') != DEMO_SUBMIT_MODE: # invalid submissions under demo mode submission can be processed return HttpResponseForbidden() return _process_form( request=request, domain=domain, app_id=app_id, user_id=None, authenticated=False, auth_cls=WaivedAuthContext, )