def set_xmlns_on_form(form_id, xmlns, app_build, log_file, dry_run): """ Set the xmlns on a form and all the corresponding forms in the saved builds that are copies of app. (form is an app_manager.models.Form) """ try: form_in_build = app_build.get_form(form_id) except FormNotFoundException: return if form_in_build.xmlns == "undefined" or form_in_build.source.count('xmlns="undefined"') > 0: if form_in_build.xmlns != "undefined": assert form_in_build.xmlns == xmlns xml = form_in_build.source wrapped_xml = XForm(xml) data = wrapped_xml.data_node.render().decode('utf-8') data = data.replace("undefined", xmlns, 1) wrapped_xml.instance_node.remove(wrapped_xml.data_node.xml) wrapped_xml.instance_node.append(parse_xml(data)) new_xml = wrapped_xml.render().decode('utf-8') form_in_build.source = new_xml form_in_build.form_migrated_from_undefined_xmlns = datetime.utcnow() log_file.write( "New xmlns for form {form_id} in app {app_build._id} is {new_xmlns}\n".format( form_id=form_id, app_build=app_build, new_xmlns=xmlns )) if not dry_run: app_build.save() # Magic happens on save
def setUp(self): self.is_usercase_in_use_patch = patch('corehq.apps.app_manager.models.is_usercase_in_use') self.is_usercase_in_use_mock = self.is_usercase_in_use_patch.start() self.is_usercase_in_use_mock.return_value = True self.app = Application.new_app('domain', 'New App') self.module = self.app.add_module(AdvancedModule.new_module('Fish Module', None)) self.module.case_type = 'fish' self.form = self.module.new_form('Form', 'en', self.get_xml('original').decode('utf-8')) self.other_module = self.app.add_module(AdvancedModule.new_module('Freshwater Module', lang='en')) self.other_module.case_type = 'freshwater' self.other_form = self.module.new_form('Other Form', 'en', self.get_xml('original').decode('utf-8')) self.case_index = CaseIndex( reference_id='host', relationship='extension', ) self.subcase = AdvancedOpenCaseAction( case_tag='open_freshwater_0', case_type='freshwater', case_name='Wanda', name_update=ConditionalCaseUpdate(question_path='/data/question1'), open_condition=FormActionCondition(type='always'), case_properties={'name': ConditionalCaseUpdate(question_path='/data/question1')}, case_indices=[self.case_index], ) self.form.actions.open_cases.append(self.subcase) self.xform = XForm(self.get_xml('original')) path = 'subcase_0/' self.subcase_block = XFormCaseBlock(self.xform, path)
def set_xmlns_on_form(form_id, xmlns, app_build, log_file, dry_run): """ Set the xmlns on a form and all the corresponding forms in the saved builds that are copies of app. (form is an app_manager.models.Form) """ try: form_in_build = app_build.get_form(form_id) except FormNotFoundException: return if form_in_build.xmlns == "undefined" or form_in_build.source.count( 'xmlns="undefined"') > 0: if form_in_build.xmlns != "undefined": assert form_in_build.xmlns == xmlns xml = form_in_build.source wrapped_xml = XForm(xml) data = wrapped_xml.data_node.render() data = data.replace("undefined", xmlns, 1) wrapped_xml.instance_node.remove(wrapped_xml.data_node.xml) wrapped_xml.instance_node.append(parse_xml(data)) new_xml = wrapped_xml.render() form_in_build.source = new_xml form_in_build.form_migrated_from_undefined_xmlns = datetime.utcnow() log_file.write( "New xmlns for form {form_id} in app {app_build._id} is {new_xmlns}\n" .format(form_id=form_id, app_build=app_build, new_xmlns=xmlns)) if not dry_run: app_build.save() # Magic happens on save
def fix_user_props_copy(app, module, form, form_ix, preloads, dry): updated = False xform = XForm(form.source) refs = {xform.resolve_path(ref): prop for ref, prop in six.iteritems(preloads)} for node in xform.model_node.findall("{f}setvalue"): if (node.attrib.get('ref') in refs and node.attrib.get('event') == "xforms-ready"): ref = node.attrib.get('ref') value = (node.attrib.get('value') or "").replace(" ", "") prop = refs[ref] userprop = "#user/" + prop if value == get_bad_usercase_path(module, form, prop): logger.info("%s setvalue %s -> %s", form_ix, userprop, ref) node.attrib["value"] = USERPROP_PREFIX + prop updated = True elif value != USERPROP_PREFIX + prop: logger.warn("%s %s has unexpected value: %r (not %s)", form_ix, ref, value, userprop) if updated: if dry: logger.info("updated setvalues in XML:\n%s", "\n".join(line for line in ET.tostring(xform.xml).split("\n") if "setvalue" in line)) else: save_xform(app, form, ET.tostring(xform.xml)) return updated
def fix_user_props_copy(app, module, form, form_ix, preloads, dry): updated = False xform = XForm(form.source) refs = {xform.resolve_path(ref): prop for ref, prop in preloads.items()} for node in xform.model_node.findall("{f}setvalue"): if (node.attrib.get('ref') in refs and node.attrib.get('event') == "xforms-ready"): ref = node.attrib.get('ref') value = (node.attrib.get('value') or "").replace(" ", "") prop = refs[ref] userprop = "#user/" + prop if value == get_bad_usercase_path(module, form, prop): logger.info("%s setvalue %s -> %s", form_ix, userprop, ref) node.attrib["value"] = USERPROP_PREFIX + prop updated = True elif value != USERPROP_PREFIX + prop: logger.warn("%s %s has unexpected value: %r (not %s)", form_ix, ref, value, userprop) if updated: if dry: logger.info( "updated setvalues in XML:\n%s", "\n".join(line for line in ET.tostring(xform.xml).split("\n") if "setvalue" in line)) else: save_xform(app, form, ET.tostring(xform.xml)) return updated
def migrate_preloads(app, form, preloads): xform = XForm(form.source) for kwargs in preloads: hashtag = kwargs.pop("hashtag") xform.add_case_preloads(**kwargs) refs = {path: [hashtag + case_property] for path, case_property in kwargs["preloads"].iteritems()} if form.case_references: form.case_references.load.update(refs) else: form.case_references = CaseReferences(load=refs) save_xform(app, form, ET.tostring(xform.xml))
def migrate_preloads(app, form, preload_items, form_ix, dry): xform = XForm(form.source) if form.case_references: load_refs = form.case_references.load else: load_refs = {} form.case_references = CaseReferences(load=load_refs) for hashtag, preloads in preload_items: if hashtag == "#case/": xform.add_case_preloads(preloads) elif hashtag == "#user/": xform.add_casedb() for nodeset, prop in preloads.items(): assert '/' not in prop, (app.id, form.unique_id, prop) xform.add_setvalue(ref=nodeset, value=USERPROP_PREFIX + prop) else: raise ValueError("unknown hashtag: " + hashtag) for nodeset, prop in preloads.items(): load_refs.setdefault(nodeset, []).append(hashtag + prop) logger.info("%s/%s %s setvalue %s = %s", app.domain, app._id, form_ix, nodeset, hashtag + prop) if dry: logger.info( "setvalue XML: %s", " ".join(line.strip() for line in ET.tostring(xform.xml).split("\n") if "setvalue" in line)) else: save_xform(app, form, ET.tostring(xform.xml))
def test_action_relevance(self): xform = XForm('') def condition_case(expected, type=None, question=None, answer=None, operator=None): condition = FormActionCondition(type=type, question=question, answer=answer, operator=operator) return condition, expected cases = [ (condition_case('true()', 'always')), (condition_case('false()', 'never')), (condition_case("/data/question1 = 'yes'", 'if', '/data/question1', 'yes')), (condition_case("selected(/data/question1, 'yes')", 'if', '/data/question1', 'yes', 'selected')), (condition_case("/data/question1", 'if', '/data/question1', None, 'boolean_true')), ] for case in cases: actual = xform.action_relevance(case[0]) self.assertEqual(actual, case[1])
def get_form_data_source(app, form): xform = XForm(form.source) form_name = form.default_name() questions = xform.get_questions([]) return DataSourceConfiguration( domain=app.domain, referenced_doc_type="XFormInstance", table_id=_clean_table_name(app.domain, form_name), display_name=form_name, configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns), configured_indicators=[ make_form_question_indicator(q, column_id=get_column_name(q["value"])) for q in questions ] + [make_form_meta_block_indicator(field) for field in FORM_METADATA_PROPERTIES], )
def save_xform(app, form, xml): def change_xmlns(xform, replacing): data = xform.data_node.render() xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id() data = data.replace(replacing, xmlns, 1) xform.instance_node.remove(xform.data_node.xml) xform.instance_node.append(parse_xml(data)) xml = xform.render() return xform, xml try: xform = XForm(xml) except XFormException: pass else: duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns] for duplicate in duplicates: if form == duplicate: continue else: xform, xml = change_xmlns(xform, xform.data_node.tag_xmlns) break GENERIC_XMLNS = "http://www.w3.org/2002/xforms" if not xform.data_node.tag_xmlns or xform.data_node.tag_xmlns == GENERIC_XMLNS: #no xmlns xform, xml = change_xmlns(xform, GENERIC_XMLNS) form.source = xml
def test_instance_check(self): xml = self.get_xml('missing_instances') with self.assertRaises(XFormValidationError) as cm: XForm(xml).add_missing_instances(self.domain) exception_message = str(cm.exception) self.assertIn('casebd', exception_message) self.assertIn('custom2', exception_message)
def _test_corpus(self, slug): xform_file = os.path.join(os.path.dirname(__file__), 'readable_forms', '{}.xform.xml'.format(slug)) submission_file = os.path.join(os.path.dirname(__file__), 'readable_forms', '{}.submission.json'.format(slug)) result_file = os.path.join(os.path.dirname(__file__), 'readable_forms', '{}.result.yaml'.format(slug)) with open(xform_file) as f: xform = f.read() with open(submission_file) as f: data = json.load(f) with open(result_file) as f: result = yaml.load(f) questions = get_questions_from_xform_node(XForm(xform), langs=['en']) questions = get_readable_form_data(data, questions) # Search for 'READABLE FORMS TEST' for more info # to bootstrap a test and have it print out your yaml result # uncomment this line. Ghetto but it works. # print yaml.safe_dump([json.loads(json.dumps(x.to_json())) # for x in questions]) self.assertJSONEqual( json.dumps([x.to_json() for x in questions]), json.dumps(result), msg= "Search for \"READABLE FORMS TEST\" for more info on fixing this test" )
def get_form_data_source(app, form): xform = XForm(form.source) schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, xform.data_node.tag_xmlns, only_process_current_builds=True, ) meta_properties = [ _export_column_to_ucr_indicator(c) for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES if c.label != 'form_link' ] dynamic_properties = _get_dynamic_indicators_from_export_schema(schema) form_name = form.default_name() config = DataSourceConfiguration( domain=app.domain, referenced_doc_type='XFormInstance', table_id=clean_table_name(app.domain, form_name), display_name=form_name, configured_filter=make_form_data_source_filter( xform.data_node.tag_xmlns, app.get_id), configured_indicators=meta_properties + dynamic_properties + _get_shared_indicators(), ) return _deduplicate_columns_if_necessary(config)
def __init__(self, domain, app, source_type, source_id): assert (source_type in ['case', 'form']) self.domain = domain self.app = app self.source_type = source_type # source_id is a case type of form id self.source_id = source_id if self.source_type == 'form': self.source_form = Form.get_form(self.source_id) self.source_xform = XForm(self.source_form.source) if self.source_type == 'case': prop_map = get_case_properties( self.app, [self.source_id], defaults=DEFAULT_CASE_PROPERTY_DATATYPES.keys()) self.case_properties = sorted( set(prop_map[self.source_id]) | {'closed'})
def get_questions(form): xform = XForm(form.source) prefix = '/%s/' % xform.data_node.tag_name def remove_prefix(string): if string.startswith(prefix): return string[len(prefix):] else: raise Exception() def transform_question(q): return { 'id': remove_prefix(q['value']), 'type': q['tag'], 'text': q['label'] if q['tag'] != 'hidden' else '' } return [transform_question(q) for q in xform.get_questions(langs)]
def migrate_app(self, app_id): app = Application.get(app_id) modules = [m for m in app.modules if m.module_type == 'basic'] for module in modules: forms = [f for f in module.forms if f.doc_type == 'Form'] for form in forms: preload = form.actions.case_preload.preload if preload: xform = XForm(form.source) xform.add_case_preloads(preload) save_xform(app, form, ET.tostring(xform.xml)) form.actions.load_from_form = form.actions.case_preload form.actions.case_preload = PreloadAction() app.vellum_case_management = True app.save()
def test_action_relevance(self): xform = XForm("") def condition_case(expected, type=None, question=None, answer=None, operator=None): condition = FormActionCondition(type=type, question=question, answer=answer, operator=operator) return condition, expected cases = [ (condition_case("true()", "always")), (condition_case("false()", "never")), (condition_case("/data/question1 = 'yes'", "if", "/data/question1", "yes")), (condition_case("selected(/data/question1, 'yes')", "if", "/data/question1", "yes", "selected")), ] for case in cases: actual = xform.action_relevance(case[0]) self.assertEqual(actual, case[1])
def migrate_preloads(app, form, preload_items, form_ix, dry): xform = XForm(form.source) if form.case_references: load_refs = form.case_references.load else: load_refs = {} form.case_references = CaseReferences(load=load_refs) for hashtag, preloads in preload_items: if hashtag == "#case/": xform.add_case_preloads(preloads) elif hashtag == "#user/": xform.add_casedb() for nodeset, prop in preloads.items(): assert '/' not in prop, (app.id, form.unique_id, prop) xform.add_setvalue(ref=nodeset, value=USERPROP_PREFIX + prop) else: raise ValueError("unknown hashtag: " + hashtag) for nodeset, prop in six.iteritems(preloads): load_refs.setdefault(nodeset, []).append(hashtag + prop) logger.info("%s/%s %s setvalue %s = %s", app.domain, app._id, form_ix, nodeset, hashtag + prop) if dry: logger.info("setvalue XML: %s", " ".join(line.strip() for line in ET.tostring(xform.xml).split("\n") if "setvalue" in line)) else: save_xform(app, form, ET.tostring(xform.xml))
def get_questions(form): xform = XForm(form.source) prefix = "/%s/" % xform.data_node.tag_name def remove_prefix(string): if string.startswith(prefix): return string[len(prefix) :] else: raise Exception() def transform_question(q): return { "id": remove_prefix(q["value"]), "type": q["tag"], "text": q["label"] if q["tag"] != "hidden" else "", } return [transform_question(q) for q in xform.get_questions(langs)]
def save_xform(app, form, xml): try: xform = XForm(xml) except XFormError: pass else: duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns] for duplicate in duplicates: if form == duplicate: continue else: data = xform.data_node.render() xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id() data = data.replace(xform.data_node.tag_xmlns, xmlns, 1) xform.instance_node.remove(xform.data_node.xml) xform.instance_node.append(parse_xml(data)) xml = xform.render() break form.source = xml
def get_form_data_source(app, form): xform = XForm(form.source) form_name = form.default_name() questions = xform.get_questions([]) return DataSourceConfiguration( domain=app.domain, referenced_doc_type='XFormInstance', table_id=_clean_table_name(app.domain, form_name), display_name=form_name, configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns), configured_indicators=[ make_form_question_indicator(q, column_id=get_column_name(q['value'])) for q in questions ] + [ make_form_meta_block_indicator(field) for field in FORM_METADATA_PROPERTIES ], )
def save_xform(app, form, xml): try: xform = XForm(xml) except XFormError: pass else: duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns] for duplicate in duplicates: if form == duplicate: continue else: data = xform.data_node.render() xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id( ) data = data.replace(xform.data_node.tag_xmlns, xmlns, 1) xform.instance_node.remove(xform.data_node.xml) xform.instance_node.append(parse_xml(data)) xml = xform.render() break form.source = xml
def migrate_app(self, app_id): app = Application.get(app_id) if app.vellum_case_management: logger.info('already migrated app {}'.format(app_id)) return modules = [m for m in app.modules if m.module_type == 'basic'] for module in modules: forms = [f for f in module.forms if f.doc_type == 'Form'] for form in forms: preload = form.actions.case_preload.preload if preload: xform = XForm(form.source) xform.add_case_preloads(preload) save_xform(app, form, ET.tostring(xform.xml)) form.actions.load_from_form = form.actions.case_preload form.actions.case_preload = PreloadAction() app.vellum_case_management = True app.save()
def fix_user_props_caseref(app, module, form, form_ix, dry): updated = False xform = XForm(form.source) refs = { xform.resolve_path(ref): vals for ref, vals in form.case_references.load.items() if any( v.startswith("#user/") for v in vals) } ref_warnings = [] for node in xform.model_node.findall("{f}setvalue"): if (node.attrib.get('ref') in refs and node.attrib.get('event') == "xforms-ready"): ref = node.attrib.get('ref') ref_values = refs[ref] if len(ref_values) != 1: ref_warnings.append((ref, " ".join(ref_values))) continue value = (node.attrib.get('value') or "").replace(" ", "") userprop = ref_values[0] assert userprop.startswith("#user/"), (ref, userprop) prop = userprop[len("#user/"):] if value == get_bad_usercase_path(module, form, prop): logger.info("%s setvalue %s -> %s", form_ix, userprop, ref) node.attrib["value"] = USERPROP_PREFIX + prop updated = True elif value != (USERPROP_PREFIX + prop).replace(" ", ""): ref_warnings.append((ref, "%r (%s)" % (value, userprop))) if updated: if dry: logger.info( "updated setvalues in XML:\n%s", "\n".join(line for line in ET.tostring( xform.xml, encoding='utf-8').split("\n") if "setvalue" in line)) else: save_xform(app, form, ET.tostring(xform.xml, encoding='utf-8')) if ref_warnings: for ref, ref_values in ref_warnings: logger.warning("%s %s has unexpected #user refs: %s", form_ix, ref, ref_values) return updated
def multimedia_list_download(request, domain, app_id): app = get_app(domain, app_id) include_audio = request.GET.get("audio", True) include_images = request.GET.get("images", True) strip_jr = request.GET.get("strip_jr", True) filelist = [] for m in app.get_modules(): for f in m.get_forms(): parsed = XForm(f.source) parsed.validate() if include_images: filelist.extend(parsed.image_references) if include_audio: filelist.extend(parsed.audio_references) if strip_jr: filelist = [s.replace("jr://file/", "") for s in filelist if s] response = HttpResponse() set_file_download(response, 'list.txt') response.write("\n".join(sorted(set(filelist)))) return response
def multimedia_list_download(request, domain, app_id): app = get_app(domain, app_id) include_audio = request.GET.get("audio", True) include_images = request.GET.get("images", True) strip_jr = request.GET.get("strip_jr", True) filelist = [] for m in app.get_modules(): for f in m.get_forms(): parsed = XForm(f.source) parsed.validate(version=app.application_version) if include_images: filelist.extend(parsed.image_references) if include_audio: filelist.extend(parsed.audio_references) if strip_jr: filelist = [s.replace("jr://file/", "") for s in filelist if s] response = HttpResponse() set_file_download(response, 'list.txt') response.write("\n".join(sorted(set(filelist)))) return response
def test_action_relevance(self): xform = XForm('') def condition_case(expected, type=None, question=None, answer=None, operator=None): condition = FormActionCondition( type=type, question=question, answer=answer, operator=operator ) return condition, expected cases = [ (condition_case('true()', 'always')), (condition_case('false()', 'never')), (condition_case("/data/question1 = 'yes'", 'if', '/data/question1', 'yes')), (condition_case("selected(/data/question1, 'yes')", 'if', '/data/question1', 'yes', 'selected')), ] for case in cases: actual = xform.action_relevance(case[0]) self.assertEqual(actual, case[1])
def migrate_app(self, app_id): app = Application.get(app_id) if app.vellum_case_management: logger.info('already migrated app {}'.format(app_id)) return modules = [m for m in app.modules if m.module_type == 'basic'] for module in modules: forms = [f for f in module.forms if f.doc_type == 'Form'] for form in forms: preload = form.actions.case_preload.preload if preload: if form.requires == 'case': xform = XForm(form.source) xform.add_case_preloads(preload) save_xform(app, form, ET.tostring(xform.xml)) form.case_references = {"load": {path: [case_property] for path, case_property in preload.iteritems()}} form.actions.case_preload = PreloadAction() app.vellum_case_management = True app.save()
def save_xform(app, form, xml): def change_xmlns(xform, old_xmlns, new_xmlns): data = xform.data_node.render().decode('utf-8') data = data.replace(old_xmlns, new_xmlns, 1) xform.instance_node.remove(xform.data_node.xml) xform.instance_node.append(parse_xml(data)) return xform.render() try: xform = XForm(xml) except XFormException: pass else: GENERIC_XMLNS = "http://www.w3.org/2002/xforms" # we assume form.get_unique_id() is unique across all of HQ and # therefore is suitable to create an XMLNS that will not confict # with any other form uid = form.get_unique_id() tag_xmlns = xform.data_node.tag_xmlns new_xmlns = form.xmlns or "http://openrosa.org/formdesigner/%s" % uid if not tag_xmlns or tag_xmlns == GENERIC_XMLNS: # no xmlns xml = change_xmlns(xform, GENERIC_XMLNS, new_xmlns) else: forms = [ form_ for form_ in app.get_xmlns_map().get(tag_xmlns, []) if form_.form_type != 'shadow_form' ] if len(forms) > 1 or (len(forms) == 1 and forms[0] is not form): if new_xmlns == tag_xmlns: new_xmlns = "http://openrosa.org/formdesigner/%s" % uid # form most likely created by app.copy_form(...) # or form is being updated with source copied from other form xml = change_xmlns(xform, tag_xmlns, new_xmlns) form.source = xml.decode('utf-8') if form.is_registration_form(): # For registration forms, assume that the first question is the # case name unless something else has been specified questions = form.get_questions([app.default_language]) if hasattr(form.actions, 'open_case'): path = form.actions.open_case.name_path if path: name_questions = [q for q in questions if q['value'] == path] if not len(name_questions): path = None if not path and len(questions): form.actions.open_case.name_path = questions[0]['value'] return xml
def fix_user_props_caseref(app, module, form, form_ix, dry): updated = False xform = XForm(form.source) refs = {xform.resolve_path(ref): vals for ref, vals in six.iteritems(form.case_references.load) if any(v.startswith("#user/") for v in vals)} ref_warnings = [] for node in xform.model_node.findall("{f}setvalue"): if (node.attrib.get('ref') in refs and node.attrib.get('event') == "xforms-ready"): ref = node.attrib.get('ref') ref_values = refs[ref] if len(ref_values) != 1: ref_warnings.append((ref, " ".join(ref_values))) continue value = (node.attrib.get('value') or "").replace(" ", "") userprop = ref_values[0] assert userprop.startswith("#user/"), (ref, userprop) prop = userprop[len("#user/"):] if value == get_bad_usercase_path(module, form, prop): logger.info("%s setvalue %s -> %s", form_ix, userprop, ref) node.attrib["value"] = USERPROP_PREFIX + prop updated = True elif value != (USERPROP_PREFIX + prop).replace(" ", ""): ref_warnings.append((ref, "%r (%s)" % (value, userprop))) if updated: if dry: logger.info("updated setvalues in XML:\n%s", "\n".join(line for line in ET.tostring(xform.xml).split("\n") if "setvalue" in line)) else: save_xform(app, form, ET.tostring(xform.xml)) if ref_warnings: for ref, ref_values in ref_warnings: logger.warning("%s %s has unexpected #user refs: %s", form_ix, ref, ref_values) return updated
def premature_auto_gps(build): app = Application.wrap(build) if app.build_version and app.build_version >= LooseVersion('2.14'): return for module in app.get_modules(): for form in module.get_forms(): try: built_source = app.fetch_attachment( 'files/modules-{}/forms-{}.xml'.format(module.id, form.id)) except ResourceNotFound: continue if form.get_auto_gps_capture(): return 'auto gps error' elif XForm(built_source).model_node.find("{orx}pollsensor"): return 'auto gps error'
def __init__(self, domain, app, source_type, source_id): assert (source_type in ['case', 'form']) self.domain = domain self.app = app self.source_type = source_type # source_id is a case type of form id self.source_id = source_id if self.source_type == 'form': self.source_form = Form.get_form(self.source_id) self.source_xform = XForm(self.source_form.source) if self.source_type == 'case': prop_map = get_case_properties( self.app, [self.source_id], defaults=DEFAULT_CASE_PROPERTY_DATATYPES.keys() ) self.case_properties = sorted(set(prop_map[self.source_id]) | {'closed'})
def __init__(self, domain, app, source_type, source_id): assert (source_type in ['case', 'form']) self.domain = domain self.app = app self.source_type = source_type # source_id is a case type of form id self.source_id = source_id if self.source_type == 'form': self.source_form = Form.get_form(self.source_id) self.source_xform = XForm(self.source_form.source) if self.source_type == 'case': property_builder = ParentCasePropertyBuilder( self.app, DEFAULT_CASE_PROPERTY_DATATYPES.keys() ) self.case_properties = list( property_builder.get_properties(self.source_id) | {'closed'} )
def save_xform(app, form, xml): def change_xmlns(xform, replacing): data = xform.data_node.render() xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id() data = data.replace(replacing, xmlns, 1) xform.instance_node.remove(xform.data_node.xml) xform.instance_node.append(parse_xml(data)) xml = xform.render() return xform, xml try: xform = XForm(xml) except XFormException: pass else: duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns] for duplicate in duplicates: if form == duplicate: continue else: xform, xml = change_xmlns(xform, xform.data_node.tag_xmlns) break GENERIC_XMLNS = "http://www.w3.org/2002/xforms" if not xform.data_node.tag_xmlns or xform.data_node.tag_xmlns == GENERIC_XMLNS: #no xmlns xform, xml = change_xmlns(xform, GENERIC_XMLNS) form.source = xml # For registration forms, assume that the first question is the case name # unless something else has been specified if toggles.APP_MANAGER_V2.enabled(app.domain): if form.is_registration_form(): questions = form.get_questions([app.default_language]) path = form.actions.open_case.name_path if path: name_questions = [q for q in questions if q['value'] == path] if not len(name_questions): path = None if not path and len(questions): form.actions.open_case.name_path = questions[0]['value']
def setUp(self): self.is_usercase_in_use_patch = patch('corehq.apps.app_manager.models.is_usercase_in_use') self.is_usercase_in_use_mock = self.is_usercase_in_use_patch.start() self.is_usercase_in_use_mock.return_value = True self.app = Application.new_app('domain', 'New App') self.module = self.app.add_module(AdvancedModule.new_module('Fish Module', None)) self.module.case_type = 'fish' self.form = self.module.new_form('New Form', None) self.case_index = CaseIndex( reference_id='host', relationship='extension', ) self.subcase = AdvancedOpenCaseAction( case_type='freshwater', case_name='Wanda', open_condition=FormActionCondition(type='always'), case_properties={'name': '/data/question1'}, case_indices=[self.case_index], ) self.form.actions.open_cases.append(self.subcase) self.xform = XForm(self.get_xml('original')) path = 'subcase_0/' self.subcase_block = CaseBlock(self.xform, path)
def test_strip_ignore_retain(self): before = self.get_xml('ignore_retain') after = self.get_xml('ignore_retain_stripped') xform = XForm(before) xform.strip_vellum_ns_attributes() self.assertXmlEqual(xform.render(), after)
def test_bad_calculate(self): source = self.get_xml('bad_calculate') xform = XForm(source) with self.assertRaises(EditFormValidationError): validate_xform_for_edit(xform)
class DataSourceBuilder(object): """ When configuring a report, one can use DataSourceBuilder to determine some of the properties of the required report data source, such as: - referenced doc type - filter - indicators """ def __init__(self, domain, app, source_type, source_id): assert (source_type in ['case', 'form']) self.domain = domain self.app = app self.source_type = source_type # source_id is a case type of form id self.source_id = source_id if self.source_type == 'form': self.source_form = Form.get_form(self.source_id) self.source_xform = XForm(self.source_form.source) if self.source_type == 'case': prop_map = get_case_properties( self.app, [self.source_id], defaults=DEFAULT_CASE_PROPERTY_DATATYPES.keys()) self.case_properties = sorted( set(prop_map[self.source_id]) | {'closed'}) @property @memoized def source_doc_type(self): if self.source_type == "case": return "CommCareCase" if self.source_type == "form": return "XFormInstance" @property @memoized def filter(self): """ Return the filter configuration for the DataSourceConfiguration. """ if self.source_type == "case": return make_case_data_source_filter(self.source_id) if self.source_type == "form": return make_form_data_source_filter( self.source_xform.data_node.tag_xmlns) @property @memoized def indicators(self): """ Return all the dict data source indicator configurations that could be used by a report that uses the same case type/form as this DataSourceConfiguration. """ ret = [] for prop in self.data_source_properties.values(): if prop['type'] == 'meta': ret.append( make_form_meta_block_indicator(prop['source'], prop['column_id'])) elif prop['type'] == "question": ret.append( make_form_question_indicator(prop['source'], prop['column_id'])) elif prop['type'] == 'case_property' and prop[ 'source'] == 'computed/owner_name': ret.append(make_owner_name_indicator(prop['column_id'])) elif prop['type'] == 'case_property': ret.append( make_case_property_indicator(prop['source'], prop['column_id'])) ret.append({ "display_name": "Count", "type": "count", "column_id": "count" }) return ret @property @memoized def data_source_properties(self): """ A dictionary containing the various properties that may be used as indicators or columns in the data source or report. Keys are strings that uniquely identify properties. Values are dicts representing the properties, ex: >> self.data_source_properties { "/data/question1": { "type": "question", "id": "/data/question1", "text": "Enter the child's name", "column_id": "data--question1", "source": { 'repeat': None, 'group': None, 'value': '/data/question1', 'label': 'question1', 'tag': 'input', 'type': 'Text' } }, "meta/deviceID": { "type": "meta", "id": "meta/deviceID", "text": "deviceID", "column_id": "meta--deviceID", "source": ("deviceID", "string") } } "id" is used as the value in selects/select2s in the form. Uniquely identifies questions. "column_id" is used as the column name for this indicator. There are bugs with slashes which requires this to be different from "id" "text" will be used as the visible text in selects/select2s "type" is "question", "case_property", or "meta" For questions, "source" is the dict returned by Xform.get_questions, for case properties and form metadata it is simply the name of the property. """ if self.source_type == 'case': ret = OrderedDict((cp, { 'type': 'case_property', 'id': cp, 'column_id': get_column_name(cp), 'text': cp, 'source': cp }) for cp in self.case_properties) ret['computed/owner_name'] = { 'type': 'case_property', 'id': 'computed/owner_name', 'column_id': get_column_name('computed/owner_name'), 'text': 'owner_name (computed)', 'source': 'computed/owner_name' } return ret # Note that owner_name is a special pseudo-case property. # The report builder will create a related_doc indicator based # on the owner_id of the case. if self.source_type == 'form': ret = OrderedDict() questions = self.source_xform.get_questions([]) ret.update((q['value'], { "type": "question", "id": q['value'], "column_id": get_column_name(q['value'].strip("/")), 'text': q['label'], "source": q, }) for q in questions) ret.update((p[0], { "type": "meta", "id": p[0], "column_id": get_column_name(p[0].strip("/")), 'text': p[0], "source": p, }) for p in FORM_METADATA_PROPERTIES) return ret @property @memoized def data_source_name(self): if self.source_type == 'form': return "{} (v{})".format(self.source_form.default_name(), self.app.version) if self.source_type == 'case': return "{} (v{})".format(self.source_id, self.app.version) def get_existing_match(self): return DataSourceConfiguration.view( 'userreports/data_sources_by_build_info', key=[ self.domain, self.source_doc_type, self.source_id, self.app._id, self.app.version ], reduce=False).one()
def setUp(self): self.xforms = {} for filename in ("label_form", "itext_form"): self.xforms[filename] = XForm(self.get_xml(filename)) self.xforms[filename].validate()
def __init__(self, domain, app_id, data_source_type, data_source_id): super().__init__(domain, app_id, data_source_type, data_source_id) self.source_form = self.app.get_form(self.data_source_id) self.source_xform = XForm(self.source_form.source)
def setUp(self): self.xform = XForm('')
def get_form_data_source(app, form): xform = XForm(form.source) form_name = form.default_name() def _get_indicator_data_type(data_type, options): if data_type == "date": return {"datatype": "date"} if data_type == "MSelect": return { "type": "choice_list", "select_style": DATATYPE_MAP[data_type], "choices": [ option['value'] for option in options ], } return {"datatype": "string"} def _make_indicator(question): path = question['value'].split('/') data_type = question['type'] options = question.get('options') ret = { "type": "raw", "column_id": path[-1], 'property_path': ['form'] + path[2:], "display_name": path[-1], } ret.update(_get_indicator_data_type(data_type, options)) return ret def _make_meta_block_indicator(field_name, data_type): ret = { "type": "raw", "column_id": field_name, "property_path": ['form', 'meta'] + [field_name], "display_name": field_name, } ret.update(_get_indicator_data_type(data_type, [])) return ret questions = xform.get_questions([]) return DataSourceConfiguration( domain=app.domain, referenced_doc_type='XFormInstance', table_id=_clean_table_name(app.domain, form_name), display_name=form_name, configured_filter={ "type": "property_match", "property_name": "xmlns", "property_path": [], "property_value": xform.data_node.tag_xmlns }, configured_indicators=[ _make_indicator(q) for q in questions ] + [ _make_meta_block_indicator(field[0], field[1]) for field in [ ('username', 'string'), ('userID', 'string'), ('timeStart', 'datetime'), ('timeEnd', 'datetime'), ('deviceID', 'string'), ] ], )
def test_strip_ignore_retain(self): before = self.get_xml("ignore_retain") after = self.get_xml("ignore_retain_stripped") xform = XForm(before) xform.strip_vellum_ns_attributes() self.assertXmlEqual(xform.render(), after)
def test_get_data_cleaning_data(self, questions_patch): builder = XFormBuilder() responses = OrderedDict() # Simple question builder.new_question('something', 'Something') responses['something'] = 'blue' # Skipped question - doesn't appear in repsonses, shouldn't appear in data cleaning data builder.new_question('skip', 'Skip me') # Simple group lights = builder.new_group('lights', 'Traffic Lights', data_type='group') lights.new_question('red', 'Red means') lights.new_question('green', 'Green means') responses['lights'] = OrderedDict([('red', 'stop'), ('green', 'go')]) # Simple repeat group, one response one_hit_wonders = builder.new_group('one_hit_wonders', 'One-Hit Wonders', data_type='repeatGroup') one_hit_wonders.new_question('name', 'Name') responses['one_hit_wonders'] = [ { 'name': 'A-Ha' }, ] # Simple repeat group, multiple responses snacks = builder.new_group('snacks', 'Snacks', data_type='repeatGroup') snacks.new_question('kind_of_snack', 'Kind of snack') responses['snacks'] = [ { 'kind_of_snack': 'samosa' }, { 'kind_of_snack': 'pakora' }, ] # Repeat group with nested group cups = builder.new_group('cups_of_tea', 'Cups of tea', data_type='repeatGroup') details = cups.new_group('details_of_cup', 'Details', data_type='group') details.new_question('kind_of_cup', 'Flavor') details.new_question('secret', 'Secret', data_type=None) responses['cups_of_tea'] = [ { 'details_of_cup': { 'kind_of_cup': 'green', 'secret': 'g' } }, { 'details_of_cup': { 'kind_of_cup': 'black', 'secret': 'b' } }, { 'details_of_cup': { 'kind_of_cup': 'more green', 'secret': 'mg' } }, ] xform = XForm(builder.tostring()) questions_patch.return_value = get_questions_from_xform_node( xform, ['en']) xml = FormSubmissionBuilder(form_id='123', form_properties=responses).as_xml_string() submitted_xform = submit_form_locally(xml, self.domain).xform form_data, _ = get_readable_data_for_submission(submitted_xform) question_response_map, ordered_question_values = get_data_cleaning_data( form_data, submitted_xform) expected_question_values = [ '/data/something', '/data/lights/red', '/data/lights/green', '/data/one_hit_wonders/name', '/data/snacks[1]/kind_of_snack', '/data/snacks[2]/kind_of_snack', '/data/cups_of_tea[1]/details_of_cup/kind_of_cup', '/data/cups_of_tea[1]/details_of_cup/secret', '/data/cups_of_tea[2]/details_of_cup/kind_of_cup', '/data/cups_of_tea[2]/details_of_cup/secret', '/data/cups_of_tea[3]/details_of_cup/kind_of_cup', '/data/cups_of_tea[3]/details_of_cup/secret', ] self.assertListEqual(ordered_question_values, expected_question_values) expected_response_map = { '/data/something': 'blue', '/data/lights/red': 'stop', '/data/lights/green': 'go', '/data/one_hit_wonders/name': 'A-Ha', '/data/snacks[1]/kind_of_snack': 'samosa', '/data/snacks[2]/kind_of_snack': 'pakora', '/data/cups_of_tea[1]/details_of_cup/kind_of_cup': 'green', '/data/cups_of_tea[1]/details_of_cup/secret': 'g', '/data/cups_of_tea[2]/details_of_cup/kind_of_cup': 'black', '/data/cups_of_tea[2]/details_of_cup/secret': 'b', '/data/cups_of_tea[3]/details_of_cup/kind_of_cup': 'more green', '/data/cups_of_tea[3]/details_of_cup/secret': 'mg', } self.assertDictEqual( {k: v['value'] for k, v in question_response_map.items()}, expected_response_map)
class CaseBlockIndexRelationshipTest(SimpleTestCase, TestXmlMixin): file_path = 'data', 'extension_case' def setUp(self): self.is_usercase_in_use_patch = patch('corehq.apps.app_manager.models.is_usercase_in_use') self.is_usercase_in_use_mock = self.is_usercase_in_use_patch.start() self.is_usercase_in_use_mock.return_value = True self.app = Application.new_app('domain', 'New App') self.module = self.app.add_module(AdvancedModule.new_module('Fish Module', None)) self.module.case_type = 'fish' self.form = self.module.new_form('New Form', None) self.case_index = CaseIndex( reference_id='host', relationship='extension', ) self.subcase = AdvancedOpenCaseAction( case_type='freshwater', case_name='Wanda', open_condition=FormActionCondition(type='always'), case_properties={'name': '/data/question1'}, case_indices=[self.case_index], ) self.form.actions.open_cases.append(self.subcase) self.xform = XForm(self.get_xml('original')) path = 'subcase_0/' self.subcase_block = CaseBlock(self.xform, path) def add_subcase_block(self): parent_node = self.xform.data_node action = next(self.form.actions.get_open_actions()) case_id = session_var(action.case_session_var) subcase_node = _make_elem('{x}subcase_0') parent_node.append(subcase_node) subcase_node.insert(0, self.subcase_block.elem) self.subcase_block.add_create_block( relevance=self.xform.action_relevance(action.open_condition), case_name=self.subcase.case_name, case_type=self.subcase.case_type, delay_case_id=bool(self.subcase.repeat_context), autoset_owner_id=autoset_owner_id_for_advanced_action(action), has_case_sharing=self.form.get_app().case_sharing, case_id=case_id ) self.subcase_block.add_update_block(self.subcase.case_properties) def test_xform_case_block_index_supports_relationship(self): """ CaseBlock index should allow the relationship to be set """ self.add_subcase_block() self.subcase_block.add_index_ref( 'host', self.form.get_case_type(), self.xform.resolve_path("case/@case_id"), self.subcase.case_indices[0].relationship, ) self.assertXmlEqual(self.get_xml('open_subcase'), str(self.xform)) def test_xform_case_block_index_default_relationship(self): """ CaseBlock index relationship should default to "child" """ child = CaseIndex( reference_id='host', relationship='child', ) self.subcase.case_indices = [child] self.add_subcase_block() self.subcase_block.add_index_ref( 'host', self.form.get_case_type(), self.xform.resolve_path("case/@case_id"), ) self.assertXmlEqual(self.get_xml('open_subcase_child'), str(self.xform)) def test_xform_case_block_index_valid_relationship(self): """ CaseBlock index relationship should only allow valid values """ with self.assertRaisesRegexp(CaseError, 'Valid values for an index relationship are "child" and "extension"'): self.subcase_block.add_index_ref( 'host', self.form.get_case_type(), self.xform.resolve_path("case/@case_id"), 'cousin', )
def __init__(self, domain, data_source_type, data_source_id): super(FormDataSourceMeta, self).__init__(domain, data_source_type, data_source_id) self.source_form = Form.get_form(self.data_source_id) self.source_xform = XForm(self.source_form.source)
class DataSourceBuilder(object): """ When configuring a report, one can use DataSourceBuilder to determine some of the properties of the required report data source, such as: - referenced doc type - filter - indicators """ def __init__(self, domain, app, source_type, source_id): assert (source_type in ['case', 'form']) self.domain = domain self.app = app self.source_type = source_type # source_id is a case type of form id self.source_id = source_id if self.source_type == 'form': self.source_form = Form.get_form(self.source_id) self.source_xform = XForm(self.source_form.source) if self.source_type == 'case': property_builder = ParentCasePropertyBuilder( self.app, DEFAULT_CASE_PROPERTY_DATATYPES.keys() ) self.case_properties = list( property_builder.get_properties(self.source_id) | {'closed'} ) @property @memoized def source_doc_type(self): if self.source_type == "case": return "CommCareCase" if self.source_type == "form": return "XFormInstance" @property @memoized def filter(self): """ Return the filter configuration for the DataSourceConfiguration. """ if self.source_type == "case": return make_case_data_source_filter(self.source_id) if self.source_type == "form": return make_form_data_source_filter(self.source_xform.data_node.tag_xmlns) @property @memoized def indicators(self): """ Return all the dict data source indicator configurations that could be used by a report that uses the same case type/form as this DataSourceConfiguration. """ ret = [] for prop in self.data_source_properties.values(): if prop['type'] == 'meta': ret.append(make_form_meta_block_indicator( prop['source'], prop['column_id'] )) elif prop['type'] == "question": ret.append(make_form_question_indicator( prop['source'], prop['column_id'] )) elif prop['type'] == 'case_property': ret.append(make_case_property_indicator( prop['source'], prop['column_id'] )) ret.append({ "display_name": "Count", "type": "count", "column_id": "count" }) return ret @property @memoized def data_source_properties(self): """ A dictionary containing the various properties that may be used as indicators or columns in the data source or report. Keys are strings that uniquely identify properties. Values are dicts representing the properties, ex: >> self.data_source_properties { "/data/question1": { "type": "question", "id": "/data/question1", "text": "Enter the child's name", "column_id": "data--question1", "source": { 'repeat': None, 'group': None, 'value': '/data/question1', 'label': 'question1', 'tag': 'input', 'type': 'Text' } }, "meta/deviceID": { "type": "meta", "id": "meta/deviceID", "text": "deviceID", "column_id": "meta--deviceID", "source": ("deviceID", "string") } } "id" is used as the value in selects/select2s in the form. Uniquely identifies questions. "column_id" is used as the column name for this indicator. There are bugs with slashes which requires this to be different from "id" "text" will be used as the visible text in selects/select2s "type" is "question", "case_property", or "meta" For questions, "source" is the dict returned by Xform.get_questions, for case properties and form metadata it is simply the name of the property. """ if self.source_type == 'case': return { cp: { 'type': 'case_property', 'id': cp, 'column_id': get_column_name(cp), 'text': cp, 'source': cp } for cp in self.case_properties } if self.source_type == 'form': ret = {} questions = self.source_xform.get_questions([]) ret.update({ q['value']: { "type": "question", "id": q['value'], "column_id": get_column_name(q['value'].strip("/")), 'text': q['label'], "source": q, } for q in questions }) ret.update({ p[0]: { "type": "meta", "id": p[0], "column_id": get_column_name(p[0].strip("/")), 'text': p[0], "source": p, } for p in FORM_METADATA_PROPERTIES }) return ret @property @memoized def data_source_name(self): if self.source_type == 'form': return "{} (v{})".format(self.source_form.default_name(), self.app.version) if self.source_type == 'case': return "{} (v{})".format(self.source_id, self.app.version) def get_existing_match(self): return DataSourceConfiguration.view( 'userreports/data_sources_by_build_info', key=[ self.domain, self.source_doc_type, self.source_id, self.app._id, self.app.version ], reduce=False ).one()
def setUp(self): self.xforms = {} for filename in ("label_form", "itext_form"): xml = self.get_xml(filename) self.xforms[filename] = XForm(xml)