def test_build_from_saved_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(len(schema.group_schemas), 1)
self.assertEqual(schema.last_app_versions[app._id], self.first_build.version)
# After the first schema has been saved let's add a second app to process
second_build = Application.wrap(self.get_json('basic_application'))
second_build._id = '456'
second_build.copy_of = app.get_id
second_build.version = 6
second_build.has_submissions = True
second_build.save()
self.addCleanup(second_build.delete)
new_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(new_schema._id, schema._id)
self.assertEqual(new_schema.last_app_versions[app._id], second_build.version)
self.assertEqual(len(new_schema.group_schemas), 1)
def test_rebuild_version_control(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
existing_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(schema._id, existing_schema._id)
with patch(
'corehq.apps.export.models.new.DATA_SCHEMA_VERSION',
DATA_SCHEMA_VERSION + 1):
rebuilt_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertNotEqual(schema._id, rebuilt_schema._id)
self.assertEqual(schema.version, DATA_SCHEMA_VERSION)
self.assertEqual(rebuilt_schema.version, DATA_SCHEMA_VERSION + 1)
def test_rebuild_version_control(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
existing_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(schema._id, existing_schema._id)
with patch(
'corehq.apps.export.models.new.FORM_DATA_SCHEMA_VERSION',
FORM_DATA_SCHEMA_VERSION + 1):
rebuilt_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertNotEqual(schema._id, rebuilt_schema._id)
self.assertEqual(schema.version, FORM_DATA_SCHEMA_VERSION)
self.assertEqual(rebuilt_schema.version, FORM_DATA_SCHEMA_VERSION + 1)
def test_build_from_saved_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(len(schema.group_schemas), 1)
self.assertEqual(schema.last_app_versions[app._id], app.version)
# After the first schema has been saved let's add a second app to process
second_build = Application.wrap(self.get_json('basic_application'))
second_build._id = '456'
second_build.copy_of = app.get_id
second_build.version = 6
second_build.save()
self.addCleanup(second_build.delete)
new_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(new_schema._id, schema._id)
self.assertEqual(new_schema.last_app_versions[app._id], app.version)
self.assertEqual(len(new_schema.group_schemas), 1)
def _get_merged_schema(self, form_name1, form_name2):
form_xml = self.get_xml(form_name1)
form_xml2 = self.get_xml(form_name2)
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml), ['en'], self.app_id, 1)
schema2 = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml2), ['en'], self.app_id, 2)
return FormExportDataSchema._merge_schemas(schema, schema2)
def test_process_app_failure(self):
'''
This ensures that the schema generated will not fail if there is an error processing one of the
applications.
'''
with patch(
'corehq.apps.export.models.new.FormExportDataSchema._process_app_build',
side_effect=Exception('boom')):
FormExportDataSchema.generate_schema_from_builds(
self.current_app.domain, self.current_app._id,
'my_sweet_xmlns')
def test_process_app_failure(self):
'''
This ensures that the schema generated will not fail if there is an error processing one of the
applications.
'''
with patch(
'corehq.apps.export.models.new.FormExportDataSchema._process_app_build',
side_effect=Exception('boom')):
FormExportDataSchema.generate_schema_from_builds(
self.current_app.domain,
self.current_app._id,
'my_sweet_xmlns'
)
def _get_merged_schema(self, form_name1, form_name2):
form_xml = self.get_xml(form_name1)
form_xml2 = self.get_xml(form_name2)
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
1
)
schema2 = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml2),
['en'],
2
)
return FormExportDataSchema._merge_schemas(schema, schema2)
def handle(self, **options):
schemas_to_rebuild = defaultdict(list)
for doc_id, domain, app_id, xmlns in _latest_form_schema_ids():
schema = FormExportDataSchema.get(doc_id)
group_schemas = schema.group_schemas[1:]
for gs in group_schemas:
if not gs.path[-1].is_repeat:
schemas_to_rebuild[domain].append((app_id, xmlns))
break
for domain, schema_keys in schemas_to_rebuild.items():
print("Rebuilding {} schemas for domain '{}'".format(len(schema_keys), domain))
for app_id, xmlns in schema_keys:
print(" rebuilding ('{}', '{}')".format(app_id, xmlns))
FormExportDataSchema.generate_schema_from_builds(domain, app_id, xmlns, force_rebuild=True)
def test_basic_application_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain, app._id, 'my_sweet_xmlns')
self.assertEqual(len(schema.group_schemas), 1)
def get_form_data_source(app, form):
xform = XForm(form.source)
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
xform.data_node.tag_xmlns,
only_process_current_builds=True,
)
meta_properties = [
_export_column_to_ucr_indicator(c)
for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES if c.label != 'form_link'
]
dynamic_properties = _get_dynamic_indicators_from_export_schema(schema)
form_name = form.default_name()
config = DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(
xform.data_node.tag_xmlns, app.get_id),
configured_indicators=meta_properties + dynamic_properties +
_get_shared_indicators(),
)
return _deduplicate_columns_if_necessary(config)
def test_xform_parsing_with_stock_questions(self):
form_xml = self.get_xml('stock_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
[],
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 6)
self.assertTrue(all(map(lambda item: item.doc_type == 'StockItem', group_schema.items)))
for parent_attr in ['@type', '@entity-id', '@date', '@section-id']:
self.assertTrue(any(map(
lambda item: item.path == [
PathNode(name='form'),
PathNode(name='balance:balance_one'),
PathNode(name=parent_attr),
],
group_schema.items,
)))
for entry_attr in ['@id', '@quantity']:
self.assertTrue(any(map(
lambda item: item.path == [
PathNode(name='form'),
PathNode(name='balance:balance_one'),
PathNode(name='entry'),
PathNode(name=entry_attr),
],
group_schema.items,
)))
def test_xform_parsing_with_repeat_group(self):
form_xml = self.get_xml('repeat_group_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
[],
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
self.assertEqual(group_schema.path, MAIN_TABLE)
form_items = filter(lambda item: item.tag is None, group_schema.items)
self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')])
self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='zendquestion')])
group_schema = schema.group_schemas[1]
self.assertEqual(len(group_schema.items), 1)
self.assertEqual(
group_schema.path,
[PathNode(name='form'), PathNode(name='question3', is_repeat=True)]
)
self.assertEqual(
group_schema.items[0].path,
[PathNode(name='form'), PathNode(name='question3', is_repeat=True), PathNode(name='question4')]
)
def setUpClass(cls):
super(TestSingleNodeRepeatConversion, cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[],
last_occurrences={cls.app_id: 2},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='single_answer')
],
label='Single Answer',
last_occurrences={cls.app_id: 2},
)
],
last_occurrences={cls.app_id: 2},
),
])
def setUpClass(cls):
super(TestConversionOrdering, cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[PathNode(name='form'), PathNode(name='question1')],
label='q1',
),
ExportItem(
path=[PathNode(name='form'), PathNode(name='question2')],
label='q2',
),
ExportItem(
path=[PathNode(name='form'), PathNode(name='other')],
label='other',
),
ExportItem(
path=[PathNode(name='form'), PathNode(name='question3')],
label='q3',
),
],
last_occurrences={cls.app_id: 2},
),
]
)
def test_xform_parsing_with_stock_questions(self):
form_xml = self.get_xml('stock_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 6)
self.assertTrue(all([item.doc_type == 'StockItem' for item in group_schema.items]))
for parent_attr in ['@type', '@entity-id', '@date', '@section-id']:
self.assertTrue(any(map(
lambda item: item.path == [
PathNode(name='form'),
PathNode(name='balance:balance_one'),
PathNode(name=parent_attr),
],
group_schema.items,
)))
for entry_attr in ['@id', '@quantity']:
self.assertTrue(any(map(
lambda item: item.path == [
PathNode(name='form'),
PathNode(name='balance:balance_one'),
PathNode(name='entry'),
PathNode(name=entry_attr),
],
group_schema.items,
)))
def test_xform_parsing_with_repeat_group(self):
form_xml = self.get_xml('repeat_group_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
self.assertEqual(group_schema.path, MAIN_TABLE)
form_items = [item for item in group_schema.items if item.tag is None]
self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')])
self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='zendquestion')])
group_schema = schema.group_schemas[1]
self.assertEqual(len(group_schema.items), 1)
self.assertEqual(
group_schema.path,
[PathNode(name='form'), PathNode(name='question3', is_repeat=True)]
)
self.assertEqual(
group_schema.items[0].path,
[PathNode(name='form'), PathNode(name='question3', is_repeat=True), PathNode(name='question4')]
)
def test_build_with_advanced_app(self):
app = self.advanced_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
"repeat-xmlns",
)
group_schema = schema.group_schemas[1] # The repeat schema
# Assert that all proper case attributes are added to advanced forms that open
# cases with repeats
assertContainsExportItems([
('form.question3.question4', 'question4'),
('form.question3.case.create.case_name',
'case_open_case_0.create.case_name'),
('form.question3.case.create.case_type',
'case_open_case_0.create.case_type'),
('form.question3.case.create.owner_id',
'case_open_case_0.create.owner_id'),
('form.question3.case.index.parent.#text',
'case_open_case_0.index.#text'),
('form.question3.case.index.parent.@case_type',
'case_open_case_0.index.@case_type'),
('form.question3.case.@case_id', 'case_open_case_0.@case_id'),
('form.question3.case.@date_modified',
'case_open_case_0.@date_modified'),
('form.question3.case.@user_id', 'case_open_case_0.@user_id'),
], group_schema)
path_suffixes = set(
[item.path[-1].name for item in group_schema.items])
self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)),
len(CASE_ATTRIBUTES))
def test_build_with_advanced_app(self):
app = self.advanced_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
"repeat-xmlns",
)
group_schema = schema.group_schemas[1] # The repeat schema
# Assert that all proper case attributes are added to advanced forms that open
# cases with repeats
assertContainsExportItems(
[
('form.question3.question4', 'question4'),
('form.question3.case.create.case_name', 'case_open_case_0.create.case_name'),
('form.question3.case.create.case_type', 'case_open_case_0.create.case_type'),
('form.question3.case.create.owner_id', 'case_open_case_0.create.owner_id'),
('form.question3.case.index.parent.#text', 'case_open_case_0.index.#text'),
('form.question3.case.index.parent.@case_type', 'case_open_case_0.index.@case_type'),
('form.question3.case.@case_id', 'case_open_case_0.@case_id'),
('form.question3.case.@date_modified', 'case_open_case_0.@date_modified'),
('form.question3.case.@user_id', 'case_open_case_0.@user_id'),
],
group_schema
)
path_suffixes = set([item.path[-1].name for item in group_schema.items])
self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)), len(CASE_ATTRIBUTES))
def get(self, request, *args, **kwargs):
app_id = request.GET.get("app_id")
xmlns = request.GET.get("export_tag").strip('"')
schema = FormExportDataSchema.generate_schema_from_builds(self.domain, app_id, xmlns, force_rebuild=True)
self.export_instance = self.export_instance_cls.generate_instance_from_schema(schema)
return super(CreateNewCustomFormExportView, self).get(request, *args, **kwargs)
def setUpClass(cls):
super(TestExportDBAccessors, cls).setUpClass()
cls.form_schema = FormExportDataSchema(
domain=cls.domain,
app_id=cls.app_id,
xmlns=cls.xmlns,
)
cls.form_schema_other = FormExportDataSchema(
domain='other',
app_id=cls.app_id,
xmlns=cls.xmlns,
)
cls.form_schema_before = FormExportDataSchema(
domain=cls.domain,
app_id=cls.app_id,
xmlns=cls.xmlns,
created_on=datetime.utcnow() - timedelta(1)
)
cls.case_schema = CaseExportDataSchema(
domain=cls.domain,
case_type=cls.case_type,
)
cls.case_schema_other = CaseExportDataSchema(
domain=cls.domain,
case_type='other',
)
cls.case_schema_before = CaseExportDataSchema(
domain=cls.domain,
case_type=cls.case_type,
created_on=datetime.utcnow() - timedelta(1)
)
cls.schemas = [
cls.form_schema,
cls.form_schema_before,
cls.form_schema_other,
cls.case_schema_before,
cls.case_schema,
cls.case_schema_other,
]
for schema in cls.schemas:
schema.save()
def test_basic_application_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(len(schema.group_schemas), 1)
def test_basic_application_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'b68a311749a6f45bdfda015b895d607012c91613'
)
self.assertEqual(len(schema.group_schemas), 1)
def test_repeat_subcases_schema_generation(self):
form_xml = self.get_xml('nested_repeat_form')
repeats_with_subcases = [
OpenSubCaseAction(repeat_context='/data/repeat',
case_properties={
'weight': '/data/repeat/group/weight',
}),
OpenSubCaseAction(repeat_context='/data/repeat/nested_repeat',
case_properties={
'age': '/data/repeat/nested_repeat/age',
}),
]
schema = FormExportDataSchema._generate_schema_from_repeat_subcases(
XForm(form_xml),
repeats_with_subcases,
['en'],
self.app_id,
1,
)
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
attribute_items = filter(
lambda item: item.path[-1].name in CASE_ATTRIBUTES,
group_schema.items)
self.assertEqual(len(attribute_items), len(CASE_ATTRIBUTES))
self.assertTrue(
all(
map(
lambda item: item.readable_path.startswith(
'form.repeat.case'),
attribute_items,
)))
create_items = filter(
lambda item: item.path[-1].name in CASE_CREATE_ELEMENTS,
group_schema.items)
self.assertEqual(len(create_items), len(CASE_CREATE_ELEMENTS))
self.assertTrue(
all(
map(
lambda item: item.readable_path.startswith(
'form.repeat.case.create'),
create_items,
)))
update_items = list(
set(group_schema.items) - set(create_items) - set(attribute_items))
self.assertEqual(len(update_items), 1)
self.assertEqual(update_items[0].readable_path,
'form.repeat.case.update.group.weight')
def test_basic_delayed_schema(self):
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=True)
self.assertIsNone(schema.last_app_versions.get(self.current_app._id))
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=False)
self.assertEqual(schema.last_app_versions[self.current_app._id],
self.build.version)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 3)
def test_basic_delayed_schema(self):
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=True
)
self.assertIsNone(schema.last_app_versions.get(self.current_app._id))
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=False
)
self.assertEqual(schema.last_app_versions[self.current_app._id], self.build.version)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 3)
def test_build_with_advanced_app(self):
app = self.advanced_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
"repeat-xmlns",
)
group_schema = schema.group_schemas[1] # The repeat schema
# Assert that all proper case attributes are added to advanced forms that open
# cases with repeats
path_suffixes = set(map(lambda item: item.path[-1].name, group_schema.items))
self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)), len(CASE_ATTRIBUTES))
def test_labels_in_xform(self):
form_xml = self.get_xml('form_with_labels')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml), ['en'], self.app_id, 1)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 1)
self.assertEqual(group_schema.items[0].path,
[PathNode(name='form'),
PathNode(name='label')])
self.assertIsInstance(group_schema.items[0], LabelItem)
def setUpClass(cls):
super(TestExportInstanceGenerationMultipleApps, cls).setUpClass()
cls.app_id = '1234'
cls.second_app_id = '5678'
cls.schema = FormExportDataSchema(group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question1')
],
label='Question 1',
last_occurrences={
cls.app_id: 2,
cls.second_app_id: 4
},
)
],
last_occurrences={
cls.app_id: 2,
cls.second_app_id: 4,
},
),
ExportGroupSchema(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='q2')
],
label='Question 2',
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
], )
def setUpClass(cls):
super(TestForceConvertFormExport, cls).setUpClass()
cls.project = create_domain(cls.domain)
cls.project.commtrack_enabled = True
cls.project.save()
cls.schema = FormExportDataSchema(
domain=cls.domain,
app_id='123',
xmlns='myxmlns',
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[],
),
],
)
def _latest_form_schema_ids():
db = FormExportDataSchema.get_db()
seen = set()
for row in db.view('schemas_by_xmlns_or_case_type/view', reduce=False, descending=True):
key_ = row['key']
doc_type = key_[1]
if doc_type != 'FormExportDataSchema':
continue
domain, doc_type, app_id, xmlns, created_on = key_
doc_key = (domain, app_id, xmlns)
if doc_key in seen:
continue
seen.add(doc_key)
yield row['id'], domain, app_id, xmlns
def test_basic_xform_parsing(self):
form_xml = self.get_xml('basic_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
1
)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
self.assertEqual(group_schema.items[0].path, ['data', 'question1'])
self.assertEqual(group_schema.items[1].path, ['data', 'question2'])
def test_repeat_subcases_schema_generation(self):
module = Module(case_type='child', _parent=Application())
form = Form().with_id(0, module)
form.actions.subcases = [
OpenSubCaseAction(
repeat_context='/data/repeat',
case_properties={
'weight':
ConditionalCaseUpdate(
question_path='/data/repeat/group/weight'),
},
subcase_index=0,
_nest=True).with_id(0, None),
OpenSubCaseAction(
repeat_context='/data/repeat',
case_properties={
'height':
ConditionalCaseUpdate(question_path='/data/repeat/height'),
},
subcase_index=1,
_nest=True).with_id(1, None),
OpenSubCaseAction(
repeat_context='/data/repeat/nested_repeat',
case_properties={
'age':
ConditionalCaseUpdate(
question_path='/data/repeat/nested_repeat/age'),
},
subcase_index=2,
_nest=False).with_id(2, None),
]
schema = FormExportDataSchema._add_export_items_for_cases(
ExportGroupSchema(path=MAIN_TABLE),
[form],
['/data/repeat', '/data/nested_repeat'],
)[0]
self.assertEqual(len(schema.group_schemas), len(form.actions.subcases))
for group_schema, action in zip(schema.group_schemas,
form.actions.subcases):
base_path = 'form.{}'.format(action.repeat_context[6:].replace(
'/', '.'))
if action._nest:
base_path += '.{}'.format(action.form_element_name)
self._check_subcase_repeat_group_schema(
group_schema, list(action.case_properties), base_path)
def test_xform_parsing_with_multiple_choice(self):
form_xml = self.get_xml('multiple_choice_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
1
)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
self.assertEqual(group_schema.items[0].path, ['data', 'question1'])
self.assertEqual(group_schema.items[1].path, ['data', 'question2'])
self.assertEqual(group_schema.items[1].options[0].value, 'choice1')
self.assertEqual(group_schema.items[1].options[1].value, 'choice2')
def get_properties_by_xmlns(domain, app_id, xmlns):
complete_xmlns = 'http://openrosa.org/formdesigner/' + xmlns
form_export_schema = get_latest_form_export_schema(
domain, app_id,
complete_xmlns) or FormExportDataSchema.generate_schema_from_builds(
domain, app_id, complete_xmlns)
if not form_export_schema.group_schemas:
return set()
else:
export_items = [
item for item in form_export_schema.group_schemas[0].items
if isinstance(item, ExportItem)
]
return set([
get_odata_property_from_export_item(item) for item in export_items
]) - {''}
def test_labels_in_xform(self):
form_xml = self.get_xml('form_with_labels')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 1)
self.assertEqual(group_schema.items[0].path, [PathNode(name='form'), PathNode(name='label')])
self.assertIsInstance(group_schema.items[0], LabelItem)
def setUpClass(cls):
super(TestFormExportInstanceGeneration, cls).setUpClass()
cls.app_id = '1234'
cls.schema = FormExportDataSchema(group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
MultipleChoiceItem(
path=[
PathNode(name='data'),
PathNode(name='question1')
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
StockItem(
path=[
PathNode(name='data'),
PathNode(name='balance:question-id'),
PathNode(name='@type'),
],
label='Stock 1',
last_occurrences={cls.app_id: 3},
)
],
last_occurrences={cls.app_id: 3},
),
ExportGroupSchema(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='q2')
],
label='Question 2',
last_occurrences={cls.app_id: 2},
),
],
last_occurrences={cls.app_id: 2},
),
], )
def test_repeat_subcases_schema_generation(self):
form_xml = self.get_xml('nested_repeat_form')
repeats_with_subcases = [
OpenSubCaseAction(
repeat_context='/data/repeat',
case_properties={
'weight': '/data/repeat/group/weight',
}
),
OpenSubCaseAction(
repeat_context='/data/repeat/nested_repeat',
case_properties={
'age': '/data/repeat/nested_repeat/age',
}
),
]
schema = FormExportDataSchema._generate_schema_from_repeat_subcases(
XForm(form_xml),
repeats_with_subcases,
['en'],
self.app_id,
1,
)
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
attribute_items = filter(lambda item: item.path[-1].name in CASE_ATTRIBUTES, group_schema.items)
self.assertEqual(len(attribute_items), len(CASE_ATTRIBUTES))
self.assertTrue(all(map(
lambda item: item.readable_path.startswith('form.repeat.case'),
attribute_items,
)))
create_items = filter(lambda item: item.path[-1].name in CASE_CREATE_ELEMENTS, group_schema.items)
self.assertEqual(len(create_items), len(CASE_CREATE_ELEMENTS))
self.assertTrue(all(map(
lambda item: item.readable_path.startswith('form.repeat.case.create'),
create_items,
)))
update_items = list(set(group_schema.items) - set(create_items) - set(attribute_items))
self.assertEqual(len(update_items), 1)
self.assertEqual(update_items[0].readable_path, 'form.repeat.case.update.group.weight')
def test_xform_parsing_with_multiple_choice(self):
form_xml = self.get_xml('multiple_choice_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml), [], ['en'], self.app_id, 1)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
form_items = filter(lambda item: item.tag is None, group_schema.items)
self.assertEqual(form_items[0].path,
[PathNode(name='form'),
PathNode(name='question1')])
self.assertEqual(form_items[1].path,
[PathNode(name='form'),
PathNode(name='question2')])
self.assertEqual(form_items[1].options[0].value, 'choice1')
self.assertEqual(form_items[1].options[1].value, 'choice2')
def test_xform_parsing_with_multiple_choice(self):
form_xml = self.get_xml('multiple_choice_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2 + len(MAIN_TABLE_PROPERTIES))
form_items = filter(lambda item: item.tag is None, group_schema.items)
self.assertEqual(form_items[0].path, ['form', 'question1'])
self.assertEqual(form_items[1].path, ['form', 'question2'])
self.assertEqual(form_items[1].options[0].value, 'choice1')
self.assertEqual(form_items[1].options[1].value, 'choice2')
def test_basic_xform_parsing(self):
form_xml = self.get_xml('basic_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml), ['en'], self.app_id, 1)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
form_items = [item for item in group_schema.items if item.tag is None]
self.assertEqual(form_items[0].path,
[PathNode(name='form'),
PathNode(name='question1')])
self.assertEqual(form_items[1].path,
[PathNode(name='form'),
PathNode(name='question2')])
def test_basic_xform_parsing(self):
form_xml = self.get_xml('basic_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 1)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
form_items = [item for item in group_schema.items if item.tag is None]
self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')])
self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='question2')])
def test_repeat_subcases_schema_generation(self):
module = Module(case_type='child', _parent=Application())
form = Form().with_id(0, module)
form.actions.subcases = [
OpenSubCaseAction(
repeat_context='/data/repeat',
case_properties={
'weight': '/data/repeat/group/weight',
},
subcase_index=0,
_nest=True
).with_id(0, None),
OpenSubCaseAction(
repeat_context='/data/repeat',
case_properties={
'height': '/data/repeat/height',
},
subcase_index=1,
_nest=True
).with_id(1, None),
OpenSubCaseAction(
repeat_context='/data/repeat/nested_repeat',
case_properties={
'age': '/data/repeat/nested_repeat/age',
},
subcase_index=2,
_nest=False
).with_id(2, None),
]
schema = FormExportDataSchema._add_export_items_for_cases(
ExportGroupSchema(path=MAIN_TABLE),
[form],
['/data/repeat', '/data/nested_repeat'],
)[0]
self.assertEqual(len(schema.group_schemas), len(form.actions.subcases))
for group_schema, action in zip(schema.group_schemas, form.actions.subcases):
base_path = 'form.{}'.format(action.repeat_context[6:].replace('/', '.'))
if action._nest:
base_path += '.{}'.format(action.form_element_name)
self._check_subcase_repeat_group_schema(group_schema, list(action.case_properties), base_path)
def setUpClass(cls):
super(TestConvertStockFormExport, cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
StockItem(
path=[
PathNode(name='form'),
PathNode(name='transfer:questionid'),
PathNode(name='entry'),
PathNode(name='@id'),
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 2},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True)
],
items=[
StockItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='transfer:questionid'),
PathNode(name='entry'),
PathNode(name='@id'),
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 2},
),
])
def setUpClass(cls):
super(TestConvertMissingAppID, cls).setUpClass()
factory = AppFactory(cls.domain)
m1, m1f1 = factory.new_basic_module('open_case', 'house')
m1f1.xmlns = 'missing-xmlns'
cls.app = factory.app
cls.app.save()
cls.project = create_domain(cls.domain)
cls.schema = FormExportDataSchema(
domain=cls.domain,
app_id='123',
xmlns='myxmlns',
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[],
),
],
)
def test_xform_parsing_with_repeat_group(self):
form_xml = self.get_xml('repeat_group_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
1
)
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 1)
self.assertEqual(group_schema.path, None)
self.assertEqual(group_schema.items[0].path, ['data', 'question1'])
group_schema = schema.group_schemas[1]
self.assertEqual(len(group_schema.items), 1)
self.assertEqual(group_schema.path, ['data', 'question3'])
self.assertEqual(group_schema.items[0].path, ['data', 'question3', 'question4'])
def test_xform_parsing_with_repeat_group(self):
form_xml = self.get_xml('repeat_group_form')
schema = FormExportDataSchema._generate_schema_from_xform(
XForm(form_xml),
['en'],
self.app_id,
1
)
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 1 + len(MAIN_TABLE_PROPERTIES))
self.assertEqual(group_schema.path, MAIN_TABLE)
form_items = filter(lambda item: item.tag is None, group_schema.items)
self.assertEqual(form_items[0].path, ['form', 'question1'])
group_schema = schema.group_schemas[1]
self.assertEqual(len(group_schema.items), 1)
self.assertEqual(group_schema.path, ['form', 'question3'])
self.assertEqual(group_schema.items[0].path, ['form', 'question3', 'question4'])
def get_form_data_source(app, form):
xform = XForm(form.source)
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
xform.data_node.tag_xmlns,
only_process_current_builds=True,
)
meta_properties = [
_export_column_to_ucr_indicator(c) for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES
if c.label != 'form_link'
]
dynamic_properties = _get_dynamic_indicators_from_export_schema(schema)
form_name = form.default_name()
config = DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns, app.get_id),
configured_indicators=meta_properties + dynamic_properties + _get_shared_indicators(),
)
return _deduplicate_columns_if_necessary(config)
def test(self):
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.app._id,
self.form_xmlns,
only_process_current_builds=True,
)
for group_schema in schema.group_schemas:
# group_schema is an instance of ExportGroupSchem
# group_schema.items is an array of ExportItems subclasses
path = [node.name for node in group_schema.path]
if path == []:
main_group_schema = group_schema
elif path == ['form', 'babies']:
baby_repeat_group_schema = group_schema
assertContainsExportItems(
[
# Verify that a simple form question appears in the schema
('form.how_are_you_today', 'How are you today?'),
('form.how_many_babies', 'How many babies?'),
('form.add_a_prescription', 'Add a prescription?'),
('form.voucher-name', '#form/voucher-name'),
('form.is_this_a_delivery', 'Is this a delivery?'),
('form.facility_name', '#form/facility_name'),
# Verify that the main parent case updates appear (case type "mom")
('form.case.@case_id', 'case.@case_id'),
('form.case.@date_modified', 'case.@date_modified'),
('form.case.@user_id', 'case.@user_id'),
('form.case.update.last_status', 'case.update.last_status'),
# Verify that we see case updates for save-to-case cases
# These are already in the form schema, so they get interpreted like any other question
('form.prescription.prescription.case.close',
'#form/prescription/prescription/case/close'),
('form.prescription.prescription.case.create.case_name',
'#form/prescription/prescription/case/create/case_name'),
('form.prescription.prescription.case.create.case_type',
'#form/prescription/prescription/case/create/case_type'),
('form.prescription.prescription.case.update.number_of_babies',
'#form/prescription/prescription/case/update/number_of_babies'
),
('form.prescription.prescription_name',
'#form/prescription/prescription_name'),
('form.prescription.prescription.case.index.parent',
'#form/prescription/prescription/case/index/parent'),
('form.prescription.prescription.case.@case_id',
'#form/prescription/prescription/case/@case_id'),
('form.prescription.prescription.case.@user_id',
'#form/prescription/prescription/case/@user_id'),
('form.prescription.prescription.case.@date_modified',
'#form/prescription/prescription/case/@date_modified'),
# # Verify that we see updates from subcases not in repeat groups (case type "voucher")
('form.subcase_0.case.@case_id', 'subcase_0.@case_id'),
('form.subcase_0.case.@date_modified',
'subcase_0.@date_modified'),
('form.subcase_0.case.@user_id', 'subcase_0.@user_id'),
('form.subcase_0.case.create.case_name',
'subcase_0.create.case_name'),
('form.subcase_0.case.create.case_type',
'subcase_0.create.case_type'),
('form.subcase_0.case.create.owner_id',
'subcase_0.create.owner_id'),
('form.subcase_0.case.index.parent.#text',
'subcase_0.index.#text'),
('form.subcase_0.case.index.parent.@case_type',
'subcase_0.index.@case_type'),
('form.subcase_0.case.update.how_many_babies',
'subcase_0.update.how_many_babies'),
],
main_group_schema)
# Verify that we see updates from subcases in repeat groups (case type "baby")
assertContainsExportItems([
('form.babies.case.@case_id', 'subcase_1.@case_id'),
('form.babies.case.@date_modified', 'subcase_1.@date_modified'),
('form.babies.case.@user_id', 'subcase_1.@user_id'),
('form.babies.case.create.case_name',
'subcase_1.create.case_name'),
('form.babies.case.create.case_type',
'subcase_1.create.case_type'),
('form.babies.case.create.owner_id', 'subcase_1.create.owner_id'),
('form.babies.case.update.eye_color',
'subcase_1.update.eye_color'),
('form.babies.case.update.facility_name',
'subcase_1.update.facility_name'),
('form.babies.case.index.parent.#text', 'subcase_1.index.#text'),
('form.babies.case.index.parent.@case_type',
'subcase_1.index.@case_type'),
('form.babies.eye_color', 'Eye color?'),
('form.babies.whats_the_babys_name', "What's the baby's name?"),
], baby_repeat_group_schema)
instance = FormExportInstance.generate_instance_from_schema(schema)
instance.export_format = Format.JSON
# make everything show up in the export
for table in instance.tables:
table.selected = True
for column in table.columns:
if column.item.path[
-1].name == '@case_id' and not column.item.transform:
self.assertFalse(column.is_advanced)
column.selected = True
with patch('corehq.apps.export.export.get_export_documents') as docs:
docs.return_value = self.form_es_response
export_data = get_export_json(instance)
def get_form_data(table):
headers = export_data[table]['headers']
return [
dict(zip(headers, row)) for row in export_data[table]['rows']
]
self.assertDictContainsSubset(
{
# normal form questions
"form.add_a_prescription": "yes_then_close",
"form.how_are_you_today": "fine_thanks",
"form.how_many_babies": "2",
"form.is_this_a_delivery": "yes",
"form.voucher-name": "Petunia2017-08-29",
# standard case update
"form.case.update.last_status": "fine_thanks",
"form.case.@case_id": "71626d9c-2d05-491f-81d9-becf8566618a",
"form.case.@user_id": "853a24735ba89a3019ced7e3153dc60d",
# save-to-case properties
"form.prescription.prescription.case.close": "True",
"form.prescription.prescription.case.create.case_name":
"Petunia-prescription-2017-08-29",
"form.prescription.prescription.case.create.case_type":
"prescription",
"form.prescription.prescription.case.update.number_of_babies":
"2",
"form.prescription.prescription_name":
"Petunia-prescription-2017-08-29",
"form.prescription.prescription.case.index.parent":
"71626d9c-2d05-491f-81d9-becf8566618a",
# non-repeating subcase actions
"form.subcase_0.case.@case_id":
"16954d55-a9be-40dd-98a8-dc7fae9c7ed6",
"form.subcase_0.case.@date_modified": "2017-08-29 11:19:40",
"form.subcase_0.case.@user_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.subcase_0.case.create.case_name": "Petunia2017-08-29",
"form.subcase_0.case.create.case_type": "voucher",
"form.subcase_0.case.create.owner_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.subcase_0.case.update.how_many_babies": "2",
"form.subcase_0.case.index.parent.#text":
"71626d9c-2d05-491f-81d9-becf8566618a",
"form.subcase_0.case.index.parent.@case_type": "mom",
},
get_form_data('Forms')[0])
self.assertDictEqual(
{
"number": "0.0",
"number__0": 0,
"number__1": 0,
"form.babies.eye_color": "brown",
"form.babies.whats_the_babys_name": "Bob",
"form.babies.case.@case_id":
"5539bd9d-d5d6-44c8-8f78-6915f16b6907",
"form.babies.case.@date_modified": "2017-08-29 11:19:40",
"form.babies.case.@user_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.babies.case.create.case_name": "Bob",
"form.babies.case.create.case_type": "baby",
"form.babies.case.create.owner_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.babies.case.index.parent.#text":
"71626d9c-2d05-491f-81d9-becf8566618a",
"form.babies.case.index.parent.@case_type": "mom",
"form.babies.case.update.eye_color": "brown",
"form.babies.case.update.facility_name": "test",
},
get_form_data('Repeat- babies')[0])
def setUpClass(cls):
super(TestConvertSavedExportSchemaToFormExportInstance,
cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='question1')
],
label='Question 1 Not updated',
last_occurrences={cls.app_id: 3},
),
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='deid_id')
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='deid_date')
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 3},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='question2')
],
label='Question 2',
last_occurrences={cls.app_id: 2},
)
],
last_occurrences={cls.app_id: 2},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='repeat_nested', is_repeat=True),
],
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='repeat_nested', is_repeat=True),
PathNode(name='nested'),
],
label='Nested Repeat',
last_occurrences={cls.app_id: 2},
)
],
last_occurrences={cls.app_id: 2},
),
],
)
def setUpClass(cls):
super(TestExportInstanceFromSavedInstance, cls).setUpClass()
cls.app_id = '1234'
cls.schema = FormExportDataSchema(group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question1')
],
label='Question 1',
last_occurrences={
cls.app_id: 3,
},
),
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='@case_id')
],
label='@case_id',
tag=PROPERTY_TAG_CASE,
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
], )
cls.new_schema = FormExportDataSchema(group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question1')
],
label='Question 1',
last_occurrences={
cls.app_id: 3,
},
),
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='@case_id')
],
label='@case_id',
tag=PROPERTY_TAG_CASE,
last_occurrences={
cls.app_id: 3,
},
),
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question3')
],
label='Question 3',
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
ExportGroupSchema(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='q2')
],
label='Question 2',
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
], )
def get_export_schema(self, export_instance):
return FormExportDataSchema.generate_schema_from_builds(
self.domain, export_instance.app_id, export_instance.xmlns
)
