def testXFormPillowListCaseProcess(self): """ Test that xform pillow can process and cleanup a single xform with a list of cases in it """ xform = XFORM_MULTI_CASES changed = transform_xform_for_elasticsearch(xform) changed_cases = extract_case_blocks(changed) orig_cases = extract_case_blocks(xform) [self.assertIsNotNone(x['@date_modified']) for x in orig_cases] [self.assertIsNone(x.get('@date_modified')) for x in changed_cases]
def testXFormPillowListCaseProcess(self): """ Test that xform pillow can process and cleanup a single xform with a list of cases in it """ xform = XFORM_MULTI_CASES pillow = XFormPillow(create_index=False, online=False) changed = pillow.change_transform(xform) changed_cases = extract_case_blocks(changed) orig_cases = extract_case_blocks(xform) [self.assertIsNotNone(x["@date_modified"]) for x in orig_cases] [self.assertIsNone(x["@date_modified"]) for x in changed_cases]
def test_parsing_date_modified(self): """ To ensure that extract_case_blocks processes date_modified when passed as datetime.datetime object using FormAccessors and form_data when at validate_phone_datetime """ xml_data = self.get_xml('create') final_xml = self._formatXForm(CREATE_XFORM_ID, xml_data, {}) response, form, cases = submit_form_locally( final_xml, TEST_DOMAIN_NAME, attachments={}, last_sync_token=None, received_on=None ) xform = FormAccessors(TEST_DOMAIN_NAME).get_form(form.get_id) extract_case_blocks(xform.form_data)
def change_transform(self, doc_dict): domain = self.get_domain(doc_dict) if domain is None: #If the domain is still None (especially when doing updates via the _changes feed) #skip and do nothing #the reason being is that changes on the xform instance do not necessarily add #domain to it, so we need to wait until the domain is at least populated before #going through with indexing this xform return None if domain not in getattr(settings, 'ES_XFORM_FULL_INDEX_DOMAINS', []): #full indexing is only enabled for select domains on an opt-in basis return None doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if doc_ret['form']['meta'].get('timeEnd', None) == "": doc_ret['form']['meta']['timeEnd'] = None if doc_ret['form']['meta'].get('timeStart', None) == "": doc_ret['form']['meta']['timeStart'] = None case_blocks = extract_case_blocks(doc_ret) for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if case_dict.get(date_modified_key, None) == "": case_dict[date_modified_key] = None #after basic transforms for stupid type mistakes are done, walk all properties. convert_properties(doc_ret['form'], self.default_mapping['properties']['form'], override_root_keys=['case']) return doc_ret
def change_transform(self, doc_dict): if self.get_domain(doc_dict) is None: #If the domain is still None (especially when doing updates via the _changes feed) #skip and do nothing #the reason being is that changes on the xform instance do not necessarily add #domain to it, so we need to wait until the domain is at least populated before #going through with indexing this xform return None else: doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if doc_ret['form']['meta'].get('timeEnd', None) == "": doc_ret['form']['meta']['timeEnd'] = None if doc_ret['form']['meta'].get('timeStart', None) == "": doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text') #see: extract_case_blocks(doc_dict) case_blocks = extract_case_blocks(doc_ret) for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if case_dict.get(date_modified_key, None) == "": case_dict[date_modified_key] = None return doc_ret
def change_transform(self, doc_dict): if self.get_domain(doc_dict) is None: # If the domain is still None (especially when doing updates via the _changes feed) # skip and do nothing # the reason being is that changes on the xform instance do not necessarily add # domain to it, so we need to wait until the domain is at least populated before # going through with indexing this xform return None else: doc_ret = copy.deepcopy(doc_dict) if "meta" in doc_ret["form"]: if doc_ret["form"]["meta"].get("timeEnd", None) == "": doc_ret["form"]["meta"]["timeEnd"] = None if doc_ret["form"]["meta"].get("timeStart", None) == "": doc_ret["form"]["meta"]["timeStart"] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret["form"]["meta"].get("appVersion"), dict): doc_ret["form"]["meta"]["appVersion"] = doc_ret["form"]["meta"]["appVersion"].get("#text") case_blocks = extract_case_blocks(doc_ret) for case_dict in case_blocks: for date_modified_key in ["date_modified", "@date_modified"]: if case_dict.get(date_modified_key, None) == "": case_dict[date_modified_key] = None # convert all mapped dict properties to nulls if they are empty strings for object_key in ["index", "attachment", "create", "update"]: if object_key in case_dict and not isinstance(case_dict[object_key], dict): case_dict[object_key] = None return doc_ret
def allowed_to_forward(self, payload): """ Forward the payload if ... * it did not come from OpenMRS, and * CaseRepeater says it's OK for the case types and users of any of the payload's cases, and * this repeater forwards to the right OpenMRS server for any of the payload's cases. :param payload: An XFormInstance (not a case) """ if payload.xmlns == XMLNS_OPENMRS: # payload came from OpenMRS. Don't send it back. return False case_blocks = extract_case_blocks(payload) case_ids = [case_block['@case_id'] for case_block in case_blocks] cases = CaseAccessors(payload.domain).get_cases(case_ids, ordered=True) if not any(CaseRepeater.allowed_to_forward(self, case) for case in cases): # If none of the case updates in the payload are allowed to # be forwarded, drop it. return False repeaters = [repeater for case in cases for repeater in get_case_location_ancestor_repeaters(case)] if repeaters and self not in repeaters: # This repeater points to the wrong OpenMRS server for this # payload. Let the right repeater handle it. return False return True
def is_case_updated(self, submission, method="couch"): # use the same case processing utilities the case code does def _case_ids_in_couch(submission): case_view = CommCareCase.get_db().view('case/by_xform_id', key=submission['_id'], reduce=False).all() return [row['id'] for row in case_view] def _case_ids_in_es(submission): query = { "filter": { "and": [ {"term": {"xform_ids": submission['_id']}} ] }, "from": 0, "size":1 } es_results = self.es['hqcases'].post('_search', data=query) return [row['_source']['_id'] for row in es_results['hits']['hits']] \ if es_results['hits']['hits'] else [] case_blocks = extract_case_blocks(submission) case_updates = [case_update_from_block(case_block) for case_block in case_blocks] case_ids_in_form = set(cu.id for cu in case_updates) case_ids_in_db = set({ "couch": _case_ids_in_couch, "es": _case_ids_in_es, }[method](submission)) missing = case_ids_in_form - case_ids_in_db return list(case_ids_in_form), list(missing), bool(missing)
def test_simple_path(self): block = {CASE_ATTR_ID: uuid.uuid4().hex} block_with_path = extract_case_blocks({ 'data': { 'some': 'stuff' }, 'case': block }, include_path=True)[0] self.assertEqual(block, block_with_path.caseblock) self.assertEqual([], block_with_path.path)
def test_basic(self): block = {CASE_ATTR_ID: uuid.uuid4().hex} blocks = extract_case_blocks({ 'data': { 'some': 'stuff' }, 'case': block }) self.assertEqual(1, len(blocks)) self.assertEqual(block, blocks[0])
def test_blocks_in_repeat(self): blocks = [{CASE_ATTR_ID: uuid.uuid4().hex} for i in range(3)] repeats = [{'group': {'case': block}} for block in blocks] blocks_back = extract_case_blocks({ 'data': { 'parent': { 'repeats': repeats } } }, include_path=True) self.assertEqual(3, len(blocks)) for i in range(len(blocks)): self.assertEqual(blocks[i], blocks_back[i].caseblock) self.assertEqual(['data', 'parent', 'repeats', 'group'], blocks_back[i].path)
def test_nested_path(self): block = {CASE_ATTR_ID: uuid.uuid4().hex} blocks = extract_case_blocks({ 'data': { 'parents': { 'parent': { 'case': block } } } }, include_path=True) self.assertEqual(1, len(blocks)) self.assertEqual(block, blocks[0].caseblock) self.assertEqual(['data', 'parents', 'parent'], blocks[0].path)
def test_blocks_in_list(self): blocks = [{CASE_ATTR_ID: uuid.uuid4().hex} for i in range(3)] blocks_back = extract_case_blocks({ 'data': { 'parent': { 'parent': { 'case': blocks } } } }, include_path=True) self.assertEqual(3, len(blocks)) for i in range(len(blocks)): self.assertEqual(blocks[i], blocks_back[i].caseblock) self.assertEqual(['data', 'parent', 'parent'], blocks_back[i].path)
def get_case_history(case): from casexml.apps.case.xform import extract_case_blocks from corehq.apps.reports.display import xmlns_to_name changes = defaultdict(dict) for form in FormAccessors(case.domain).get_forms(case.xform_ids): case_blocks = extract_case_blocks(form) for block in case_blocks: if block.get('@case_id') == case.case_id: property_changes = { 'Form ID': form.form_id, 'Form Name': xmlns_to_name(case.domain, form.xmlns, form.app_id), 'Form Received On': form.received_on, 'Form Submitted By': form.metadata.username, } property_changes.update(block.get('create', {})) property_changes.update(block.get('update', {})) changes[form.form_id].update(property_changes) return sorted(six.itervalues(changes), key=lambda f: f['Form Received On'])
def __call__(self, item, context=None): case_id = self._case_id_expression(item, context) if not case_id: return [] forms = self._case_forms_expression(item, context) cache_key = (self.__class__.__name__, case_id) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) case_history = [] for f in forms: case_blocks = extract_case_blocks(f) case_history.append( next(case_block for case_block in case_blocks if case_block['@case_id'] == case_id)) context.set_cache_value(cache_key, case_history) return case_history
def transform_xform_for_elasticsearch(doc_dict, include_props=True): """ Given an XFormInstance, return a copy that is ready to be sent to elasticsearch, or None, if the form should not be saved to elasticsearch """ if doc_dict.get('domain', None) is None: # if there is no domain don't bother processing it return None else: doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)): doc_ret['form']['meta']['timeEnd'] = None if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)): doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text') case_blocks = extract_case_blocks(doc_ret) for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if not is_valid_date(case_dict.get(date_modified_key, None)): if case_dict.get(date_modified_key) == '': case_dict[date_modified_key] = None else: case_dict.pop(date_modified_key, None) # convert all mapped dict properties to nulls if they are empty strings for object_key in ['index', 'attachment', 'create', 'update']: if object_key in case_dict and not isinstance(case_dict[object_key], dict): case_dict[object_key] = None doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict)) if include_props: form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()] doc_ret["__props_for_querying"] = form_props return doc_ret
def change_transform(self, doc_dict, include_props=True): if self.get_domain(doc_dict) is None: #If the domain is still None (especially when doing updates via the _changes feed) #skip and do nothing #the reason being is that changes on the xform instance do not necessarily add #domain to it, so we need to wait until the domain is at least populated before #going through with indexing this xform return None else: doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)): doc_ret['form']['meta']['timeEnd'] = None if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)): doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text') case_blocks = extract_case_blocks(doc_ret) for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if not is_valid_date(case_dict.get(date_modified_key, None)): if case_dict.get(date_modified_key) == '': case_dict[date_modified_key] = None else: case_dict.pop(date_modified_key, None) # convert all mapped dict properties to nulls if they are empty strings for object_key in ['index', 'attachment', 'create', 'update']: if object_key in case_dict and not isinstance(case_dict[object_key], dict): case_dict[object_key] = None doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict)) if include_props: form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()] doc_ret["__props_for_querying"] = form_props return doc_ret
def extract_form_info(form, properties=None, case_ids=case_ids): unknown_number = 0 meta = form["form"].get("meta", dict()) # get case ids case_blocks = extract_case_blocks(form) cases = {c["@case_id"] for c in case_blocks} case_ids |= cases form_info = { "form": form, "attachments": list(), "name": form["form"].get("@name", "unknown form"), "user": meta.get("username", "unknown_user"), "cases": cases, "id": form["_id"], } for k, v in form["_attachments"].iteritems(): if v["content_type"] == "text/xml": continue try: question_id = unicode(u"-".join(find_question_id(form["form"], k))) except TypeError: question_id = unicode(u"unknown" + unicode(unknown_number)) unknown_number += 1 if not properties or question_id in properties: extension = unicode(os.path.splitext(k)[1]) form_info["attachments"].append( { "size": v["length"], "name": k, "question_id": question_id, "extension": extension, "timestamp": parse(form["received_on"]).timetuple(), } ) return form_info
def extract_form_info(form, properties=None, case_ids=case_ids): unknown_number = 0 meta = form['form'].get('meta', dict()) # get case ids case_blocks = extract_case_blocks(form) cases = {c['@case_id'] for c in case_blocks} case_ids |= cases form_info = { 'form': form, 'attachments': list(), 'name': form['form'].get('@name', 'unknown form'), 'user': meta.get('username', 'unknown_user'), 'cases': cases, 'id': form['_id'] } for k, v in form['_attachments'].iteritems(): if v['content_type'] == 'text/xml': continue try: question_id = unicode(u'-'.join(find_question_id(form['form'], k))) except TypeError: question_id = unicode(u'unknown' + unicode(unknown_number)) unknown_number += 1 if not properties or question_id in properties: extension = unicode(os.path.splitext(k)[1]) form_info['attachments'].append({ 'size': v['length'], 'name': k, 'question_id': question_id, 'extension': extension, 'timestamp': parse(form['received_on']).timetuple(), }) return form_info
def transform_xform_for_elasticsearch(doc_dict): """ Given an XFormInstance, return a copy that is ready to be sent to elasticsearch, or None, if the form should not be saved to elasticsearch """ doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)): doc_ret['form']['meta']['timeEnd'] = None if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)): doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta'] = format_form_meta_for_es(doc_ret['form']['meta']) app_version_info = get_app_version_info( doc_ret['domain'], doc_ret.get('build_id'), doc_ret.get('version'), doc_ret['form']['meta'], ) doc_ret['form']['meta']['commcare_version'] = app_version_info.commcare_version doc_ret['form']['meta']['app_build_version'] = app_version_info.build_version try: geo_point = GeoPointProperty().wrap(doc_ret['form']['meta']['location']) doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon except (KeyError, BadValueError): doc_ret['form']['meta']['geo_point'] = None pass try: user_id = doc_ret['form']['meta']['userID'] except KeyError: user_id = None doc_ret['user_type'] = get_user_type(user_id) doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat() try: case_blocks = extract_case_blocks(doc_ret) except PhoneDateValueError: pass else: for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if not is_valid_date(case_dict.get(date_modified_key, None)): if case_dict.get(date_modified_key) == '': case_dict[date_modified_key] = None else: case_dict.pop(date_modified_key, None) # convert all mapped dict properties to nulls if they are empty strings for object_key in ['index', 'attachment', 'create', 'update']: if object_key in case_dict and not isinstance(case_dict[object_key], dict): case_dict[object_key] = None try: doc_ret["__retrieved_case_ids"] = list(set(case_update_from_block(cb).id for cb in case_blocks)) except CaseGenerationException: doc_ret["__retrieved_case_ids"] = [] if 'backend_id' not in doc_ret: doc_ret['backend_id'] = 'couch' return doc_ret
def get_case_ids_for_case_upload(case_upload): for form_record in case_upload.form_records.order_by('pk').all(): form = FormAccessors(case_upload.domain).get_form(form_record.form_id) for case_block in extract_case_blocks(form): yield case_block['@case_id']
def render_form(form, domain, options): """ Uses options since Django 1.3 doesn't seem to support templatetag kwargs. Change to kwargs when we're on a version of Django that does. """ # don't actually use the passed in timezone since we assume form submissions already come # in in local time. # todo: we should revisit this when we properly handle timezones in form processing. timezone = pytz.utc case_id = options.get('case_id') readable_form_data = READABLE_FORM_DATA.enabled(domain) use_old_reason = '' case_id_attr = "@%s" % const.CASE_TAG_ID _get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone) form_dict = form.top_level_tags() form_dict.pop('change', None) # this data already in Case Changes tab # Form Data tab if readable_form_data: try: form_data = get_readable_form_data(form) except QuestionListNotFound as e: readable_form_data = False use_old_reason = e.message if not readable_form_data: form_keys = [k for k in form_dict.keys() if form_key_filter(k)] form_data = _get_tables_as_columns(form_dict, get_definition(form_keys)) # Case Changes tab case_blocks = extract_case_blocks(form) for i, block in enumerate(list(case_blocks)): if case_id and block.get(case_id_attr) == case_id: case_blocks.pop(i) case_blocks.insert(0, block) cases = [] for b in case_blocks: this_case_id = b.get(case_id_attr) try: this_case = CommCareCase.get(this_case_id) if this_case_id else None valid_case = True except ResourceNotFound: this_case = None valid_case = False if this_case and this_case._id: url = reverse('case_details', args=[domain, this_case._id]) else: url = "#" definition = get_definition(sorted_case_update_keys(b.keys())) cases.append({ "is_current_case": case_id and this_case_id == case_id, "name": case_inline_display(this_case), "table": _get_tables_as_columns(b, definition), "url": url, "valid_case": valid_case }) # Form Metadata tab meta = form_dict.pop('meta', {}) definition = get_definition(sorted_form_metadata_keys(meta.keys())) form_meta_data = _get_tables_as_columns(meta, definition) if 'auth_context' in form: auth_context = AuthContext(form.auth_context) auth_context_user_id = auth_context.user_id auth_user_info = get_doc_info_by_id(domain, auth_context_user_id) else: auth_user_info = get_doc_info_by_id(domain, None) auth_context = AuthContext( user_id=None, authenticated=False, domain=domain, ) meta_userID = meta.get('userID') meta_username = meta.get('username') if meta_userID == 'demo_user': user_info = DocInfo( domain=domain, display='demo_user', ) elif meta_username == 'admin': user_info = DocInfo( domain=domain, display='admin', ) else: user_info = get_doc_info_by_id(domain, meta_userID) return render_to_string("reports/form/partials/single_form.html", { "context_case_id": case_id, "instance": form, "is_archived": form.doc_type == "XFormArchived", "domain": domain, 'readable_form_data': readable_form_data, 'use_old_reason': use_old_reason, "form_data": form_data, "cases": cases, "form_table_options": { # todo: wells if display config has more than one column "put_loners_in_wells": False }, "form_meta_data": form_meta_data, "auth_context": auth_context, "auth_user_info": auth_user_info, "user_info": user_info, })
def render_form(form, domain, options): """ Uses options since Django 1.3 doesn't seem to support templatetag kwargs. Change to kwargs when we're on a version of Django that does. """ timezone = options.get('timezone', pytz.utc) #display = options.get('display') case_id = options.get('case_id') case_id_attr = "@%s" % const.CASE_TAG_ID _get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone) # Form Data tab form_dict = form.top_level_tags() form_dict.pop('change', None) # this data already in Case Changes tab form_keys = [k for k in form_dict.keys() if form_key_filter(k)] form_data = _get_tables_as_columns(form_dict, get_definition(form_keys)) # Case Changes tab case_blocks = extract_case_blocks(form) for i, block in enumerate(list(case_blocks)): if case_id and block.get(case_id_attr) == case_id: case_blocks.pop(i) case_blocks.insert(0, block) cases = [] for b in case_blocks: this_case_id = b.get(case_id_attr) try: this_case = CommCareCase.get(this_case_id) if this_case_id else None valid_case = True except ResourceNotFound: this_case = None valid_case = False if this_case and this_case._id: url = reverse('case_details', args=[domain, this_case._id]) else: url = "#" definition = get_definition(sorted_case_update_keys(b.keys())) cases.append({ "is_current_case": case_id and this_case_id == case_id, "name": case_inline_display(this_case), "table": _get_tables_as_columns(b, definition), "url": url, "valid_case": valid_case }) # Form Metadata tab meta = form_dict.pop('meta', {}) definition = get_definition(sorted_form_metadata_keys(meta.keys())) form_meta_data = _get_tables_as_columns(meta, definition) return render_to_string("form/partials/single_form.html", { "context_case_id": case_id, "instance": form, "is_archived": form.doc_type == "XFormArchived", "domain": domain, "form_data": form_data, "cases": cases, "form_table_options": { # todo: wells if display config has more than one column "put_loners_in_wells": False }, "form_meta_data": form_meta_data, })
def _extract_form_attachment_info(form, properties): """ This is a helper function for build_form_multimedia_zip. Return a dict containing information about the given form and its relevant attachments """ def find_question_id(form, value): for k, v in form.iteritems(): if isinstance(v, dict): ret = find_question_id(v, value) if ret: return [k] + ret else: if v == value: return [k] return None unknown_number = 0 case_blocks = extract_case_blocks(form.form_data) form_info = { 'form': form, 'attachments': [], 'case_ids': {c['@case_id'] for c in case_blocks}, 'username': form.get_data('form/meta/username') } # TODO make form.attachments always return objects that conform to a # uniform interface. XFormInstance attachment values are dicts, and # XFormInstanceSQL attachment values are XFormAttachmentSQL objects. for attachment_name, attachment in form.attachments.iteritems(): if hasattr(attachment, 'content_type'): content_type = attachment.content_type else: content_type = attachment['content_type'] if content_type == 'text/xml': continue try: question_id = unicode( u'-'.join(find_question_id(form.form_data, attachment_name))) except TypeError: question_id = u'unknown' + unicode(unknown_number) unknown_number += 1 if not properties or question_id in properties: extension = unicode(os.path.splitext(attachment_name)[1]) if hasattr(attachment, 'content_length'): # FormAttachmentSQL or BlobMeta size = attachment.content_length elif 'content_length' in attachment: # dict from BlobMeta.to_json() or possibly FormAttachmentSQL size = attachment['content_length'] else: # couch attachment dict size = attachment['length'] form_info['attachments'].append({ 'size': size, 'name': attachment_name, 'question_id': question_id, 'extension': extension, 'timestamp': form.received_on.timetuple(), }) return form_info
def _extract_form_attachment_info(form, properties): """ This is a helper function for build_form_multimedia_zip. Return a dict containing information about the given form and its relevant attachments """ def find_question_id(form, value): for k, v in six.iteritems(form): if isinstance(v, dict): ret = find_question_id(v, value) if ret: return [k] + ret elif isinstance(v, list): for repeat in v: ret = find_question_id(repeat, value) if ret: return [k] + ret else: if v == value: return [k] return None unknown_number = 0 case_blocks = extract_case_blocks(form.form_data) form_info = { 'form': form, 'attachments': [], 'case_ids': {c['@case_id'] for c in case_blocks}, 'username': form.get_data('form/meta/username') } # TODO make form.attachments always return objects that conform to a # uniform interface. XFormInstance attachment values are dicts, and # XFormInstanceSQL attachment values are XFormAttachmentSQL objects. for attachment_name, attachment in six.iteritems(form.attachments): if hasattr(attachment, 'content_type'): content_type = attachment.content_type else: content_type = attachment['content_type'] if content_type == 'text/xml': continue try: question_id = six.text_type('-'.join( find_question_id(form.form_data, attachment_name))) except TypeError: question_id = 'unknown' + six.text_type(unknown_number) unknown_number += 1 if not properties or question_id in properties: extension = six.text_type(os.path.splitext(attachment_name)[1]) if hasattr(attachment, 'content_length'): # FormAttachmentSQL or BlobMeta size = attachment.content_length elif 'content_length' in attachment: # dict from BlobMeta.to_json() or possibly FormAttachmentSQL size = attachment['content_length'] else: # couch attachment dict size = attachment['length'] form_info['attachments'].append({ 'size': size, 'name': attachment_name, 'question_id': question_id, 'extension': extension, 'timestamp': form.received_on.timetuple(), }) return form_info
def render_form(form, domain, options): """ Uses options since Django 1.3 doesn't seem to support templatetag kwargs. Change to kwargs when we're on a version of Django that does. """ # don't actually use the passed in timezone since we assume form submissions already come # in in local time. # todo: we should revisit this when we properly handle timezones in form processing. timezone = pytz.utc case_id = options.get('case_id') side_pane = options.get('side_pane', False) user = options.get('user', None) _get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone) # Form Data tab form_data, question_list_not_found = get_readable_data_for_submission(form) # Case Changes tab case_blocks = extract_case_blocks(form) for i, block in enumerate(list(case_blocks)): if case_id and block.get(const.CASE_ATTR_ID) == case_id: case_blocks.pop(i) case_blocks.insert(0, block) cases = [] for b in case_blocks: this_case_id = b.get(const.CASE_ATTR_ID) try: this_case = CommCareCase.get(this_case_id) if this_case_id else None valid_case = True except ResourceNotFound: this_case = None valid_case = False if this_case and this_case._id: url = reverse('case_details', args=[domain, this_case._id]) else: url = "#" definition = get_definition(sorted_case_update_keys(b.keys())) cases.append({ "is_current_case": case_id and this_case_id == case_id, "name": case_inline_display(this_case), "table": _get_tables_as_columns(b, definition), "url": url, "valid_case": valid_case }) # Form Metadata tab meta = form.top_level_tags().get('meta', None) or {} definition = get_definition(sorted_form_metadata_keys(meta.keys())) form_meta_data = _get_tables_as_columns(meta, definition) if 'auth_context' in form: auth_context = AuthContext(form.auth_context) auth_context_user_id = auth_context.user_id auth_user_info = get_doc_info_by_id(domain, auth_context_user_id) else: auth_user_info = get_doc_info_by_id(domain, None) auth_context = AuthContext( user_id=None, authenticated=False, domain=domain, ) meta_userID = meta.get('userID') meta_username = meta.get('username') if meta_userID == 'demo_user': user_info = DocInfo( domain=domain, display='demo_user', ) elif meta_username == 'admin': user_info = DocInfo( domain=domain, display='admin', ) else: user_info = get_doc_info_by_id(domain, meta_userID) request = options.get('request', None) user_can_edit = ( request and user and request.domain and (user.can_edit_data() or user.is_commcare_user()) ) show_edit_submission = ( user_can_edit and has_privilege(request, privileges.CLOUDCARE) and toggle_enabled(request, toggles.EDIT_SUBMISSIONS) ) # stuffing this in the same flag as case rebuild show_resave = ( user_can_edit and toggle_enabled(request, toggles.CASE_REBUILD) ) return render_to_string("reports/form/partials/single_form.html", { "context_case_id": case_id, "instance": form, "is_archived": form.doc_type == "XFormArchived", "domain": domain, 'question_list_not_found': question_list_not_found, "form_data": form_data, "cases": cases, "form_table_options": { # todo: wells if display config has more than one column "put_loners_in_wells": False }, "form_meta_data": form_meta_data, "auth_context": auth_context, "auth_user_info": auth_user_info, "user_info": user_info, "side_pane": side_pane, "user": user, "show_edit_submission": show_edit_submission, "show_resave": show_resave, })
def transform_xform_for_elasticsearch(doc_dict): """ Given an XFormInstance, return a copy that is ready to be sent to elasticsearch, or None, if the form should not be saved to elasticsearch """ doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)): doc_ret['form']['meta']['timeEnd'] = None if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)): doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta'] = format_form_meta_for_es( doc_ret['form']['meta']) app_version_info = get_app_version_info( doc_ret['domain'], doc_ret.get('build_id'), doc_ret.get('version'), doc_ret['form']['meta'], ) doc_ret['form']['meta'][ 'commcare_version'] = app_version_info.commcare_version doc_ret['form']['meta'][ 'app_build_version'] = app_version_info.build_version try: geo_point = GeoPointProperty().wrap( doc_ret['form']['meta']['location']) doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon except (KeyError, BadValueError): doc_ret['form']['meta']['geo_point'] = None pass try: user_id = doc_ret['form']['meta']['userID'] except KeyError: user_id = None doc_ret['user_type'] = get_user_type(user_id) doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat() try: case_blocks = extract_case_blocks(doc_ret) except PhoneDateValueError: pass else: for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if not is_valid_date(case_dict.get(date_modified_key, None)): if case_dict.get(date_modified_key) == '': case_dict[date_modified_key] = None else: case_dict.pop(date_modified_key, None) # convert all mapped dict properties to nulls if they are empty strings for object_key in ['index', 'attachment', 'create', 'update']: if object_key in case_dict and not isinstance( case_dict[object_key], dict): case_dict[object_key] = None try: doc_ret["__retrieved_case_ids"] = list( set(case_update_from_block(cb).id for cb in case_blocks)) except CaseGenerationException: doc_ret["__retrieved_case_ids"] = [] if 'backend_id' not in doc_ret: doc_ret['backend_id'] = 'couch' return doc_ret
def render_form(form, domain, options): """ Uses options since Django 1.3 doesn't seem to support templatetag kwargs. Change to kwargs when we're on a version of Django that does. """ timezone = get_timezone_for_request() case_id = options.get('case_id') side_pane = options.get('side_pane', False) user = options.get('user', None) request = options.get('request', None) support_enabled = toggle_enabled(request, toggles.SUPPORT) _get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone) # Form Data tab form_data, question_list_not_found = get_readable_data_for_submission(form) # Case Changes tab case_blocks = extract_case_blocks(form) for i, block in enumerate(list(case_blocks)): if case_id and block.get(const.CASE_ATTR_ID) == case_id: case_blocks.pop(i) case_blocks.insert(0, block) cases = [] for b in case_blocks: this_case_id = b.get(const.CASE_ATTR_ID) try: this_case = CaseAccessors(domain).get_case(this_case_id) if this_case_id else None valid_case = True except ResourceNotFound: this_case = None valid_case = False if this_case and this_case.case_id: url = reverse('case_details', args=[domain, this_case.case_id]) else: url = "#" definition = get_default_definition( sorted_case_update_keys(b.keys()), assume_phonetimes=(not form.metadata or (form.metadata.deviceID != CLOUDCARE_DEVICE_ID)), ) cases.append({ "is_current_case": case_id and this_case_id == case_id, "name": case_inline_display(this_case), "table": _get_tables_as_columns(b, definition), "url": url, "valid_case": valid_case }) # Form Metadata tab meta = _top_level_tags(form).get('meta', None) or {} meta['received_on'] = json_format_datetime(form.received_on) if support_enabled: meta['last_sync_token'] = form.last_sync_token definition = get_default_definition(sorted_form_metadata_keys(meta.keys())) form_meta_data = _get_tables_as_columns(meta, definition) if getattr(form, 'auth_context', None): auth_context = AuthContext(form.auth_context) auth_context_user_id = auth_context.user_id auth_user_info = get_doc_info_by_id(domain, auth_context_user_id) else: auth_user_info = get_doc_info_by_id(domain, None) auth_context = AuthContext( user_id=None, authenticated=False, domain=domain, ) meta_userID = meta.get('userID') meta_username = meta.get('username') if meta_userID == 'demo_user': user_info = DocInfo( domain=domain, display='demo_user', ) elif meta_username == 'admin': user_info = DocInfo( domain=domain, display='admin', ) else: user_info = get_doc_info_by_id(domain, meta_userID) user_can_edit = ( request and user and request.domain and (user.can_edit_data() or user.is_commcare_user()) ) show_edit_options = ( user_can_edit and can_edit_form_location(domain, user, form) ) show_edit_submission = ( user_can_edit and has_privilege(request, privileges.DATA_CLEANUP) and not form.is_deprecated ) show_resave = ( user_can_edit and support_enabled ) def _get_edit_info(instance): info = { 'was_edited': False, 'is_edit': False, } if instance.is_deprecated: info.update({ 'was_edited': True, 'latest_version': instance.orig_id, }) if getattr(instance, 'edited_on', None) and getattr(instance, 'deprecated_form_id', None): info.update({ 'is_edit': True, 'edited_on': instance.edited_on, 'previous_version': instance.deprecated_form_id }) return info return render_to_string("reports/form/partials/single_form.html", { "context_case_id": case_id, "instance": form, "is_archived": form.is_archived, "edit_info": _get_edit_info(form), "domain": domain, 'question_list_not_found': question_list_not_found, "form_data": form_data, "cases": cases, "form_table_options": { # todo: wells if display config has more than one column "put_loners_in_wells": False }, "form_meta_data": form_meta_data, "auth_context": auth_context, "auth_user_info": auth_user_info, "user_info": user_info, "side_pane": side_pane, "show_edit_options": show_edit_options, "show_edit_submission": show_edit_submission, "show_resave": show_resave, }, RequestContext(request))
def render_form(form, domain, options): """ Uses options since Django 1.3 doesn't seem to support templatetag kwargs. Change to kwargs when we're on a version of Django that does. """ # don't actually use the passed in timezone since we assume form submissions already come # in in local time. # todo: we should revisit this when we properly handle timezones in form processing. timezone = pytz.utc case_id = options.get('case_id') side_pane = options.get('side_pane', False) user = options.get('user', None) case_id_attr = "@%s" % const.CASE_TAG_ID _get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone) # Form Data tab form_data, question_list_not_found = get_readable_data_for_submission(form) # Case Changes tab case_blocks = extract_case_blocks(form) for i, block in enumerate(list(case_blocks)): if case_id and block.get(case_id_attr) == case_id: case_blocks.pop(i) case_blocks.insert(0, block) cases = [] for b in case_blocks: this_case_id = b.get(case_id_attr) try: this_case = CommCareCase.get(this_case_id) if this_case_id else None valid_case = True except ResourceNotFound: this_case = None valid_case = False if this_case and this_case._id: url = reverse('case_details', args=[domain, this_case._id]) else: url = "#" definition = get_definition(sorted_case_update_keys(b.keys())) cases.append({ "is_current_case": case_id and this_case_id == case_id, "name": case_inline_display(this_case), "table": _get_tables_as_columns(b, definition), "url": url, "valid_case": valid_case }) # Form Metadata tab meta = form.top_level_tags().get('meta', None) or {} definition = get_definition(sorted_form_metadata_keys(meta.keys())) form_meta_data = _get_tables_as_columns(meta, definition) if 'auth_context' in form: auth_context = AuthContext(form.auth_context) auth_context_user_id = auth_context.user_id auth_user_info = get_doc_info_by_id(domain, auth_context_user_id) else: auth_user_info = get_doc_info_by_id(domain, None) auth_context = AuthContext( user_id=None, authenticated=False, domain=domain, ) meta_userID = meta.get('userID') meta_username = meta.get('username') if meta_userID == 'demo_user': user_info = DocInfo( domain=domain, display='demo_user', ) elif meta_username == 'admin': user_info = DocInfo( domain=domain, display='admin', ) else: user_info = get_doc_info_by_id(domain, meta_userID) edit_session_data = {'user_id': meta_userID} if len(case_blocks) == 1 and case_blocks[0].get(case_id_attr): edit_session_data["case_id"] = case_blocks[0].get(case_id_attr) return render_to_string("reports/form/partials/single_form.html", { "context_case_id": case_id, "instance": form, "form_meta": options.get('form_meta', {}), "maps_api_key": settings.GMAPS_API_KEY, "is_archived": form.doc_type == "XFormArchived", "domain": domain, 'question_list_not_found': question_list_not_found, "form_data": form_data, "cases": cases, "form_table_options": { # todo: wells if display config has more than one column "put_loners_in_wells": False }, "form_meta_data": form_meta_data, "auth_context": auth_context, "auth_user_info": auth_user_info, "user_info": user_info, "side_pane": side_pane, "user": user, "edit_session_data": edit_session_data, "request": options.get('request', None), # needed for toggles })