def save_xform(app, form, xml):
def change_xmlns(xform, replacing):
data = xform.data_node.render()
xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id()
data = data.replace(replacing, xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
xml = xform.render()
return xform, xml
try:
xform = XForm(xml)
except XFormException:
pass
else:
duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns]
for duplicate in duplicates:
if form == duplicate:
continue
else:
xform, xml = change_xmlns(xform, xform.data_node.tag_xmlns)
break
GENERIC_XMLNS = "http://www.w3.org/2002/xforms"
if not xform.data_node.tag_xmlns or xform.data_node.tag_xmlns == GENERIC_XMLNS: #no xmlns
xform, xml = change_xmlns(xform, GENERIC_XMLNS)
form.source = xml
def _test_corpus(self, slug):
xform_file = os.path.join(os.path.dirname(__file__), 'readable_forms',
'{}.xform.xml'.format(slug))
submission_file = os.path.join(os.path.dirname(__file__),
'readable_forms',
'{}.submission.json'.format(slug))
result_file = os.path.join(os.path.dirname(__file__), 'readable_forms',
'{}.result.yaml'.format(slug))
with open(xform_file) as f:
xform = f.read()
with open(submission_file) as f:
data = json.load(f)
with open(result_file) as f:
result = yaml.load(f)
questions = get_questions_from_xform_node(XForm(xform), langs=['en'])
questions = get_readable_form_data(data, questions)
# Search for 'READABLE FORMS TEST' for more info
# to bootstrap a test and have it print out your yaml result
# uncomment this line. Ghetto but it works.
# print yaml.safe_dump([json.loads(json.dumps(x.to_json()))
# for x in questions])
self.assertJSONEqual(
json.dumps([x.to_json() for x in questions]),
json.dumps(result),
msg=
"Search for \"READABLE FORMS TEST\" for more info on fixing this test"
)
def set_xmlns_on_form(form_id, xmlns, app_build, log_file, dry_run):
"""
Set the xmlns on a form and all the corresponding forms in the saved builds
that are copies of app.
(form is an app_manager.models.Form)
"""
try:
form_in_build = app_build.get_form(form_id)
except FormNotFoundException:
return
if form_in_build.xmlns == "undefined" or form_in_build.source.count(
'xmlns="undefined"') > 0:
if form_in_build.xmlns != "undefined":
assert form_in_build.xmlns == xmlns
xml = form_in_build.source
wrapped_xml = XForm(xml)
data = wrapped_xml.data_node.render()
data = data.replace("undefined", xmlns, 1)
wrapped_xml.instance_node.remove(wrapped_xml.data_node.xml)
wrapped_xml.instance_node.append(parse_xml(data))
new_xml = wrapped_xml.render()
form_in_build.source = new_xml
form_in_build.form_migrated_from_undefined_xmlns = datetime.utcnow()
log_file.write(
"New xmlns for form {form_id} in app {app_build._id} is {new_xmlns}\n"
.format(form_id=form_id, app_build=app_build, new_xmlns=xmlns))
if not dry_run:
app_build.save() # Magic happens on save
def setUp(self):
self.is_usercase_in_use_patch = patch('corehq.apps.app_manager.models.is_usercase_in_use')
self.is_usercase_in_use_mock = self.is_usercase_in_use_patch.start()
self.is_usercase_in_use_mock.return_value = True
self.app = Application.new_app('domain', 'New App')
self.module = self.app.add_module(AdvancedModule.new_module('Fish Module', None))
self.module.case_type = 'fish'
self.form = self.module.new_form('Form', 'en', self.get_xml('original').decode('utf-8'))
self.other_module = self.app.add_module(AdvancedModule.new_module('Freshwater Module', lang='en'))
self.other_module.case_type = 'freshwater'
self.other_form = self.module.new_form('Other Form', 'en', self.get_xml('original').decode('utf-8'))
self.case_index = CaseIndex(
reference_id='host',
relationship='extension',
)
self.subcase = AdvancedOpenCaseAction(
case_tag='open_freshwater_0',
case_type='freshwater',
case_name='Wanda',
name_update=ConditionalCaseUpdate(question_path='/data/question1'),
open_condition=FormActionCondition(type='always'),
case_properties={'name': ConditionalCaseUpdate(question_path='/data/question1')},
case_indices=[self.case_index],
)
self.form.actions.open_cases.append(self.subcase)
self.xform = XForm(self.get_xml('original'))
path = 'subcase_0/'
self.subcase_block = XFormCaseBlock(self.xform, path)
def migrate_preloads(app, form, preload_items, form_ix, dry):
xform = XForm(form.source)
if form.case_references:
load_refs = form.case_references.load
else:
load_refs = {}
form.case_references = CaseReferences(load=load_refs)
for hashtag, preloads in preload_items:
if hashtag == "#case/":
xform.add_case_preloads(preloads)
elif hashtag == "#user/":
xform.add_casedb()
for nodeset, prop in preloads.items():
assert '/' not in prop, (app.id, form.unique_id, prop)
xform.add_setvalue(ref=nodeset, value=USERPROP_PREFIX + prop)
else:
raise ValueError("unknown hashtag: " + hashtag)
for nodeset, prop in preloads.items():
load_refs.setdefault(nodeset, []).append(hashtag + prop)
logger.info("%s/%s %s setvalue %s = %s", app.domain, app._id,
form_ix, nodeset, hashtag + prop)
if dry:
logger.info(
"setvalue XML: %s",
" ".join(line.strip()
for line in ET.tostring(xform.xml).split("\n")
if "setvalue" in line))
else:
save_xform(app, form, ET.tostring(xform.xml))
def fix_user_props_copy(app, module, form, form_ix, preloads, dry):
updated = False
xform = XForm(form.source)
refs = {xform.resolve_path(ref): prop for ref, prop in preloads.items()}
for node in xform.model_node.findall("{f}setvalue"):
if (node.attrib.get('ref') in refs
and node.attrib.get('event') == "xforms-ready"):
ref = node.attrib.get('ref')
value = (node.attrib.get('value') or "").replace(" ", "")
prop = refs[ref]
userprop = "#user/" + prop
if value == get_bad_usercase_path(module, form, prop):
logger.info("%s setvalue %s -> %s", form_ix, userprop, ref)
node.attrib["value"] = USERPROP_PREFIX + prop
updated = True
elif value != USERPROP_PREFIX + prop:
logger.warn("%s %s has unexpected value: %r (not %s)", form_ix,
ref, value, userprop)
if updated:
if dry:
logger.info(
"updated setvalues in XML:\n%s",
"\n".join(line for line in ET.tostring(xform.xml).split("\n")
if "setvalue" in line))
else:
save_xform(app, form, ET.tostring(xform.xml))
return updated
def test_action_relevance(self):
xform = XForm('')
def condition_case(expected,
type=None,
question=None,
answer=None,
operator=None):
condition = FormActionCondition(type=type,
question=question,
answer=answer,
operator=operator)
return condition, expected
cases = [
(condition_case('true()', 'always')),
(condition_case('false()', 'never')),
(condition_case("/data/question1 = 'yes'", 'if', '/data/question1',
'yes')),
(condition_case("selected(/data/question1, 'yes')", 'if',
'/data/question1', 'yes', 'selected')),
(condition_case("/data/question1", 'if', '/data/question1', None,
'boolean_true')),
]
for case in cases:
actual = xform.action_relevance(case[0])
self.assertEqual(actual, case[1])
def test_instance_check(self):
xml = self.get_xml('missing_instances')
with self.assertRaises(XFormValidationError) as cm:
XForm(xml).add_missing_instances(self.domain)
exception_message = str(cm.exception)
self.assertIn('casebd', exception_message)
self.assertIn('custom2', exception_message)
def get_form_data_source(app, form):
xform = XForm(form.source)
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
xform.data_node.tag_xmlns,
only_process_current_builds=True,
)
meta_properties = [
_export_column_to_ucr_indicator(c)
for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES if c.label != 'form_link'
]
dynamic_properties = _get_dynamic_indicators_from_export_schema(schema)
form_name = form.default_name()
config = DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(
xform.data_node.tag_xmlns, app.get_id),
configured_indicators=meta_properties + dynamic_properties +
_get_shared_indicators(),
)
return _deduplicate_columns_if_necessary(config)
def migrate_preloads(app, form, preloads):
xform = XForm(form.source)
for kwargs in preloads:
hashtag = kwargs.pop("hashtag")
xform.add_case_preloads(**kwargs)
refs = {path: [hashtag + case_property]
for path, case_property in kwargs["preloads"].iteritems()}
if form.case_references:
form.case_references.load.update(refs)
else:
form.case_references = CaseReferences(load=refs)
save_xform(app, form, ET.tostring(xform.xml))
def save_xform(app, form, xml):
def change_xmlns(xform, old_xmlns, new_xmlns):
data = xform.data_node.render().decode('utf-8')
data = data.replace(old_xmlns, new_xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
return xform.render()
try:
xform = XForm(xml)
except XFormException:
pass
else:
GENERIC_XMLNS = "http://www.w3.org/2002/xforms"
# we assume form.get_unique_id() is unique across all of HQ and
# therefore is suitable to create an XMLNS that will not confict
# with any other form
uid = form.get_unique_id()
tag_xmlns = xform.data_node.tag_xmlns
new_xmlns = form.xmlns or "http://openrosa.org/formdesigner/%s" % uid
if not tag_xmlns or tag_xmlns == GENERIC_XMLNS: # no xmlns
xml = change_xmlns(xform, GENERIC_XMLNS, new_xmlns)
else:
forms = [
form_ for form_ in app.get_xmlns_map().get(tag_xmlns, [])
if form_.form_type != 'shadow_form'
]
if len(forms) > 1 or (len(forms) == 1 and forms[0] is not form):
if new_xmlns == tag_xmlns:
new_xmlns = "http://openrosa.org/formdesigner/%s" % uid
# form most likely created by app.copy_form(...)
# or form is being updated with source copied from other form
xml = change_xmlns(xform, tag_xmlns, new_xmlns)
form.source = xml.decode('utf-8')
if form.is_registration_form():
# For registration forms, assume that the first question is the
# case name unless something else has been specified
questions = form.get_questions([app.default_language])
if hasattr(form.actions, 'open_case'):
path = form.actions.open_case.name_path
if path:
name_questions = [q for q in questions if q['value'] == path]
if not len(name_questions):
path = None
if not path and len(questions):
form.actions.open_case.name_path = questions[0]['value']
return xml
def premature_auto_gps(build):
app = Application.wrap(build)
if app.build_version and app.build_version >= LooseVersion('2.14'):
return
for module in app.get_modules():
for form in module.get_forms():
try:
built_source = app.fetch_attachment(
'files/modules-{}/forms-{}.xml'.format(module.id, form.id))
except ResourceNotFound:
continue
if form.get_auto_gps_capture():
return 'auto gps error'
elif XForm(built_source).model_node.find("{orx}pollsensor"):
return 'auto gps error'
def __init__(self, domain, app, source_type, source_id):
assert (source_type in ['case', 'form'])
self.domain = domain
self.app = app
self.source_type = source_type
# source_id is a case type of form id
self.source_id = source_id
if self.source_type == 'form':
self.source_form = Form.get_form(self.source_id)
self.source_xform = XForm(self.source_form.source)
if self.source_type == 'case':
prop_map = get_case_properties(
self.app, [self.source_id],
defaults=DEFAULT_CASE_PROPERTY_DATATYPES.keys())
self.case_properties = sorted(
set(prop_map[self.source_id]) | {'closed'})
def get_questions(form):
xform = XForm(form.source)
prefix = '/%s/' % xform.data_node.tag_name
def remove_prefix(string):
if string.startswith(prefix):
return string[len(prefix):]
else:
raise Exception()
def transform_question(q):
return {
'id': remove_prefix(q['value']),
'type': q['tag'],
'text': q['label'] if q['tag'] != 'hidden' else ''
}
return [transform_question(q) for q in xform.get_questions(langs)]
def get_form_data_source(app, form):
xform = XForm(form.source)
form_name = form.default_name()
questions = xform.get_questions([])
return DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=_clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns),
configured_indicators=[
make_form_question_indicator(q, column_id=get_column_name(q['value']))
for q in questions
] + [
make_form_meta_block_indicator(field)
for field in FORM_METADATA_PROPERTIES
],
)
def save_xform(app, form, xml):
try:
xform = XForm(xml)
except XFormError:
pass
else:
duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns]
for duplicate in duplicates:
if form == duplicate:
continue
else:
data = xform.data_node.render()
xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id(
)
data = data.replace(xform.data_node.tag_xmlns, xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
xml = xform.render()
break
form.source = xml
def migrate_app(self, app_id):
app = Application.get(app_id)
if app.vellum_case_management:
logger.info('already migrated app {}'.format(app_id))
return
modules = [m for m in app.modules if m.module_type == 'basic']
for module in modules:
forms = [f for f in module.forms if f.doc_type == 'Form']
for form in forms:
preload = form.actions.case_preload.preload
if preload:
xform = XForm(form.source)
xform.add_case_preloads(preload)
save_xform(app, form, ET.tostring(xform.xml))
form.actions.load_from_form = form.actions.case_preload
form.actions.case_preload = PreloadAction()
app.vellum_case_management = True
app.save()
def fix_user_props_caseref(app, module, form, form_ix, dry):
updated = False
xform = XForm(form.source)
refs = {
xform.resolve_path(ref): vals
for ref, vals in form.case_references.load.items() if any(
v.startswith("#user/") for v in vals)
}
ref_warnings = []
for node in xform.model_node.findall("{f}setvalue"):
if (node.attrib.get('ref') in refs
and node.attrib.get('event') == "xforms-ready"):
ref = node.attrib.get('ref')
ref_values = refs[ref]
if len(ref_values) != 1:
ref_warnings.append((ref, " ".join(ref_values)))
continue
value = (node.attrib.get('value') or "").replace(" ", "")
userprop = ref_values[0]
assert userprop.startswith("#user/"), (ref, userprop)
prop = userprop[len("#user/"):]
if value == get_bad_usercase_path(module, form, prop):
logger.info("%s setvalue %s -> %s", form_ix, userprop, ref)
node.attrib["value"] = USERPROP_PREFIX + prop
updated = True
elif value != (USERPROP_PREFIX + prop).replace(" ", ""):
ref_warnings.append((ref, "%r (%s)" % (value, userprop)))
if updated:
if dry:
logger.info(
"updated setvalues in XML:\n%s",
"\n".join(line for line in ET.tostring(
xform.xml, encoding='utf-8').split("\n")
if "setvalue" in line))
else:
save_xform(app, form, ET.tostring(xform.xml, encoding='utf-8'))
if ref_warnings:
for ref, ref_values in ref_warnings:
logger.warning("%s %s has unexpected #user refs: %s", form_ix, ref,
ref_values)
return updated
def save_xform(app, form, xml):
def change_xmlns(xform, replacing):
data = xform.data_node.render()
xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id()
data = data.replace(replacing, xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
xml = xform.render()
return xform, xml
try:
xform = XForm(xml)
except XFormException:
pass
else:
duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns]
for duplicate in duplicates:
if form == duplicate:
continue
else:
xform, xml = change_xmlns(xform, xform.data_node.tag_xmlns)
break
GENERIC_XMLNS = "http://www.w3.org/2002/xforms"
if not xform.data_node.tag_xmlns or xform.data_node.tag_xmlns == GENERIC_XMLNS: #no xmlns
xform, xml = change_xmlns(xform, GENERIC_XMLNS)
form.source = xml
# For registration forms, assume that the first question is the case name
# unless something else has been specified
if toggles.APP_MANAGER_V2.enabled(app.domain):
if form.is_registration_form():
questions = form.get_questions([app.default_language])
path = form.actions.open_case.name_path
if path:
name_questions = [q for q in questions if q['value'] == path]
if not len(name_questions):
path = None
if not path and len(questions):
form.actions.open_case.name_path = questions[0]['value']
def multimedia_list_download(request, domain, app_id):
app = get_app(domain, app_id)
include_audio = request.GET.get("audio", True)
include_images = request.GET.get("images", True)
strip_jr = request.GET.get("strip_jr", True)
filelist = []
for m in app.get_modules():
for f in m.get_forms():
validate_xform(domain, f.source)
parsed = XForm(f.source)
if include_images:
filelist.extend(parsed.image_references)
if include_audio:
filelist.extend(parsed.audio_references)
if strip_jr:
filelist = [s.replace("jr://file/", "") for s in filelist if s]
response = HttpResponse()
set_file_download(response, 'list.txt')
response.write("\n".join(sorted(set(filelist))))
return response
def migrate_app(self, app_id):
app = Application.get(app_id)
if app.vellum_case_management:
logger.info('already migrated app {}'.format(app_id))
return
modules = [m for m in app.modules if m.module_type == 'basic']
for module in modules:
forms = [f for f in module.forms if f.doc_type == 'Form']
for form in forms:
preload = form.actions.case_preload.preload
if preload:
if form.requires == 'case':
xform = XForm(form.source)
xform.add_case_preloads(preload)
save_xform(app, form, ET.tostring(xform.xml))
form.case_references = {"load": {path: [case_property]
for path, case_property in preload.iteritems()}}
form.actions.case_preload = PreloadAction()
app.vellum_case_management = True
app.save()
def test_strip_ignore_retain(self):
before = self.get_xml('ignore_retain')
after = self.get_xml('ignore_retain_stripped')
xform = XForm(before)
xform.strip_vellum_ns_attributes()
self.assertXmlEqual(xform.render(), after)
def setUp(self):
self.xform = XForm('')
def test_bad_calculate(self):
source = self.get_xml('bad_calculate')
xform = XForm(source)
with self.assertRaises(EditFormValidationError):
validate_xform_for_edit(xform)
def test_instance_check(self):
xml = self.get_xml('missing_instances')
missing_instances, missing_unknown_instances = find_missing_instances(
XForm(xml))
self.assertEqual({'ledgerdb'}, missing_instances)
self.assertEqual({'casebd', 'custom2'}, missing_unknown_instances)
def __init__(self, domain, app_id, data_source_type, data_source_id):
super().__init__(domain, app_id, data_source_type, data_source_id)
self.source_form = self.app.get_form(self.data_source_id)
self.source_xform = XForm(self.source_form.source)
def __init__(self, domain, data_source_type, data_source_id):
super(FormDataSourceMeta, self).__init__(domain, data_source_type,
data_source_id)
self.source_form = Form.get_form(self.data_source_id)
self.source_xform = XForm(self.source_form.source)
def setUp(self):
self.xforms = {}
for filename in ("label_form", "itext_form"):
self.xforms[filename] = XForm(self.get_xml(filename))
self.xforms[filename].validate()
def test_get_data_cleaning_data(self, questions_patch):
builder = XFormBuilder()
responses = OrderedDict()
# Simple question
builder.new_question('something', 'Something')
responses['something'] = 'blue'
# Skipped question - doesn't appear in repsonses, shouldn't appear in data cleaning data
builder.new_question('skip', 'Skip me')
# Simple group
lights = builder.new_group('lights',
'Traffic Lights',
data_type='group')
lights.new_question('red', 'Red means')
lights.new_question('green', 'Green means')
responses['lights'] = OrderedDict([('red', 'stop'), ('green', 'go')])
# Simple repeat group, one response
one_hit_wonders = builder.new_group('one_hit_wonders',
'One-Hit Wonders',
data_type='repeatGroup')
one_hit_wonders.new_question('name', 'Name')
responses['one_hit_wonders'] = [
{
'name': 'A-Ha'
},
]
# Simple repeat group, multiple responses
snacks = builder.new_group('snacks', 'Snacks', data_type='repeatGroup')
snacks.new_question('kind_of_snack', 'Kind of snack')
responses['snacks'] = [
{
'kind_of_snack': 'samosa'
},
{
'kind_of_snack': 'pakora'
},
]
# Repeat group with nested group
cups = builder.new_group('cups_of_tea',
'Cups of tea',
data_type='repeatGroup')
details = cups.new_group('details_of_cup',
'Details',
data_type='group')
details.new_question('kind_of_cup', 'Flavor')
details.new_question('secret', 'Secret', data_type=None)
responses['cups_of_tea'] = [
{
'details_of_cup': {
'kind_of_cup': 'green',
'secret': 'g'
}
},
{
'details_of_cup': {
'kind_of_cup': 'black',
'secret': 'b'
}
},
{
'details_of_cup': {
'kind_of_cup': 'more green',
'secret': 'mg'
}
},
]
xform = XForm(builder.tostring())
questions_patch.return_value = get_questions_from_xform_node(
xform, ['en'])
xml = FormSubmissionBuilder(form_id='123',
form_properties=responses).as_xml_string()
submitted_xform = submit_form_locally(xml, self.domain).xform
form_data, _ = get_readable_data_for_submission(submitted_xform)
question_response_map, ordered_question_values = get_data_cleaning_data(
form_data, submitted_xform)
expected_question_values = [
'/data/something',
'/data/lights/red',
'/data/lights/green',
'/data/one_hit_wonders/name',
'/data/snacks[1]/kind_of_snack',
'/data/snacks[2]/kind_of_snack',
'/data/cups_of_tea[1]/details_of_cup/kind_of_cup',
'/data/cups_of_tea[1]/details_of_cup/secret',
'/data/cups_of_tea[2]/details_of_cup/kind_of_cup',
'/data/cups_of_tea[2]/details_of_cup/secret',
'/data/cups_of_tea[3]/details_of_cup/kind_of_cup',
'/data/cups_of_tea[3]/details_of_cup/secret',
]
self.assertListEqual(ordered_question_values, expected_question_values)
expected_response_map = {
'/data/something': 'blue',
'/data/lights/red': 'stop',
'/data/lights/green': 'go',
'/data/one_hit_wonders/name': 'A-Ha',
'/data/snacks[1]/kind_of_snack': 'samosa',
'/data/snacks[2]/kind_of_snack': 'pakora',
'/data/cups_of_tea[1]/details_of_cup/kind_of_cup': 'green',
'/data/cups_of_tea[1]/details_of_cup/secret': 'g',
'/data/cups_of_tea[2]/details_of_cup/kind_of_cup': 'black',
'/data/cups_of_tea[2]/details_of_cup/secret': 'b',
'/data/cups_of_tea[3]/details_of_cup/kind_of_cup': 'more green',
'/data/cups_of_tea[3]/details_of_cup/secret': 'mg',
}
self.assertDictEqual(
{k: v['value']
for k, v in question_response_map.items()}, expected_response_map)
def setUp(self):
self.xforms = {}
for filename in ("label_form", "itext_form"):
xml = self.get_xml(filename)
self.xforms[filename] = XForm(xml)
