def test_build_from_saved_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(len(schema.group_schemas), 1)
self.assertEqual(schema.last_app_versions[app._id], app.version)
# After the first schema has been saved let's add a second app to process
second_build = Application.wrap(self.get_json('basic_application'))
second_build._id = '456'
second_build.copy_of = app.get_id
second_build.version = 6
second_build.save()
self.addCleanup(second_build.delete)
new_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(new_schema._id, schema._id)
self.assertEqual(new_schema.last_app_versions[app._id], app.version)
self.assertEqual(len(new_schema.group_schemas), 1)
def test_rebuild_version_control(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
existing_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(schema._id, existing_schema._id)
with patch(
'corehq.apps.export.models.new.DATA_SCHEMA_VERSION',
DATA_SCHEMA_VERSION + 1):
rebuilt_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertNotEqual(schema._id, rebuilt_schema._id)
self.assertEqual(schema.version, DATA_SCHEMA_VERSION)
self.assertEqual(rebuilt_schema.version, DATA_SCHEMA_VERSION + 1)
def test_rebuild_version_control(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
existing_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(schema._id, existing_schema._id)
with patch(
'corehq.apps.export.models.new.FORM_DATA_SCHEMA_VERSION',
FORM_DATA_SCHEMA_VERSION + 1):
rebuilt_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertNotEqual(schema._id, rebuilt_schema._id)
self.assertEqual(schema.version, FORM_DATA_SCHEMA_VERSION)
self.assertEqual(rebuilt_schema.version, FORM_DATA_SCHEMA_VERSION + 1)
def test_build_from_saved_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(len(schema.group_schemas), 1)
self.assertEqual(schema.last_app_versions[app._id], self.first_build.version)
# After the first schema has been saved let's add a second app to process
second_build = Application.wrap(self.get_json('basic_application'))
second_build._id = '456'
second_build.copy_of = app.get_id
second_build.version = 6
second_build.has_submissions = True
second_build.save()
self.addCleanup(second_build.delete)
new_schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(new_schema._id, schema._id)
self.assertEqual(new_schema.last_app_versions[app._id], second_build.version)
self.assertEqual(len(new_schema.group_schemas), 1)
def test_process_app_failure(self):
'''
This ensures that the schema generated will not fail if there is an error processing one of the
applications.
'''
with patch(
'corehq.apps.export.models.new.FormExportDataSchema._process_app_build',
side_effect=Exception('boom')):
FormExportDataSchema.generate_schema_from_builds(
self.current_app.domain, self.current_app._id,
'my_sweet_xmlns')
def test_process_app_failure(self):
'''
This ensures that the schema generated will not fail if there is an error processing one of the
applications.
'''
with patch(
'corehq.apps.export.models.new.FormExportDataSchema._process_app_build',
side_effect=Exception('boom')):
FormExportDataSchema.generate_schema_from_builds(
self.current_app.domain,
self.current_app._id,
'my_sweet_xmlns'
)
def handle(self, **options):
schemas_to_rebuild = defaultdict(list)
for doc_id, domain, app_id, xmlns in _latest_form_schema_ids():
schema = FormExportDataSchema.get(doc_id)
group_schemas = schema.group_schemas[1:]
for gs in group_schemas:
if not gs.path[-1].is_repeat:
schemas_to_rebuild[domain].append((app_id, xmlns))
break
for domain, schema_keys in schemas_to_rebuild.items():
print("Rebuilding {} schemas for domain '{}'".format(len(schema_keys), domain))
for app_id, xmlns in schema_keys:
print(" rebuilding ('{}', '{}')".format(app_id, xmlns))
FormExportDataSchema.generate_schema_from_builds(domain, app_id, xmlns, force_rebuild=True)
def get_form_data_source(app, form):
xform = XForm(form.source)
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
xform.data_node.tag_xmlns,
only_process_current_builds=True,
)
meta_properties = [
_export_column_to_ucr_indicator(c)
for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES if c.label != 'form_link'
]
dynamic_properties = _get_dynamic_indicators_from_export_schema(schema)
form_name = form.default_name()
config = DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(
xform.data_node.tag_xmlns, app.get_id),
configured_indicators=meta_properties + dynamic_properties +
_get_shared_indicators(),
)
return _deduplicate_columns_if_necessary(config)
def test_build_with_advanced_app(self):
app = self.advanced_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
"repeat-xmlns",
)
group_schema = schema.group_schemas[1] # The repeat schema
# Assert that all proper case attributes are added to advanced forms that open
# cases with repeats
assertContainsExportItems([
('form.question3.question4', 'question4'),
('form.question3.case.create.case_name',
'case_open_case_0.create.case_name'),
('form.question3.case.create.case_type',
'case_open_case_0.create.case_type'),
('form.question3.case.create.owner_id',
'case_open_case_0.create.owner_id'),
('form.question3.case.index.parent.#text',
'case_open_case_0.index.#text'),
('form.question3.case.index.parent.@case_type',
'case_open_case_0.index.@case_type'),
('form.question3.case.@case_id', 'case_open_case_0.@case_id'),
('form.question3.case.@date_modified',
'case_open_case_0.@date_modified'),
('form.question3.case.@user_id', 'case_open_case_0.@user_id'),
], group_schema)
path_suffixes = set(
[item.path[-1].name for item in group_schema.items])
self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)),
len(CASE_ATTRIBUTES))
def test_basic_application_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain, app._id, 'my_sweet_xmlns')
self.assertEqual(len(schema.group_schemas), 1)
def test_build_with_advanced_app(self):
app = self.advanced_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
"repeat-xmlns",
)
group_schema = schema.group_schemas[1] # The repeat schema
# Assert that all proper case attributes are added to advanced forms that open
# cases with repeats
assertContainsExportItems(
[
('form.question3.question4', 'question4'),
('form.question3.case.create.case_name', 'case_open_case_0.create.case_name'),
('form.question3.case.create.case_type', 'case_open_case_0.create.case_type'),
('form.question3.case.create.owner_id', 'case_open_case_0.create.owner_id'),
('form.question3.case.index.parent.#text', 'case_open_case_0.index.#text'),
('form.question3.case.index.parent.@case_type', 'case_open_case_0.index.@case_type'),
('form.question3.case.@case_id', 'case_open_case_0.@case_id'),
('form.question3.case.@date_modified', 'case_open_case_0.@date_modified'),
('form.question3.case.@user_id', 'case_open_case_0.@user_id'),
],
group_schema
)
path_suffixes = set([item.path[-1].name for item in group_schema.items])
self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)), len(CASE_ATTRIBUTES))
def get(self, request, *args, **kwargs):
app_id = request.GET.get("app_id")
xmlns = request.GET.get("export_tag").strip('"')
schema = FormExportDataSchema.generate_schema_from_builds(self.domain, app_id, xmlns, force_rebuild=True)
self.export_instance = self.export_instance_cls.generate_instance_from_schema(schema)
return super(CreateNewCustomFormExportView, self).get(request, *args, **kwargs)
def test_basic_application_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'b68a311749a6f45bdfda015b895d607012c91613'
)
self.assertEqual(len(schema.group_schemas), 1)
def test_basic_application_schema(self):
app = self.current_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
'my_sweet_xmlns'
)
self.assertEqual(len(schema.group_schemas), 1)
def test_basic_delayed_schema(self):
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=True)
self.assertIsNone(schema.last_app_versions.get(self.current_app._id))
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=False)
self.assertEqual(schema.last_app_versions[self.current_app._id],
self.build.version)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 3)
def test_basic_delayed_schema(self):
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=True
)
self.assertIsNone(schema.last_app_versions.get(self.current_app._id))
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.xmlns,
only_process_current_builds=False
)
self.assertEqual(schema.last_app_versions[self.current_app._id], self.build.version)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 3)
def test_build_with_advanced_app(self):
app = self.advanced_app
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
"repeat-xmlns",
)
group_schema = schema.group_schemas[1] # The repeat schema
# Assert that all proper case attributes are added to advanced forms that open
# cases with repeats
path_suffixes = set(map(lambda item: item.path[-1].name, group_schema.items))
self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)), len(CASE_ATTRIBUTES))
def get_properties_by_xmlns(domain, app_id, xmlns):
complete_xmlns = 'http://openrosa.org/formdesigner/' + xmlns
form_export_schema = get_latest_form_export_schema(
domain, app_id,
complete_xmlns) or FormExportDataSchema.generate_schema_from_builds(
domain, app_id, complete_xmlns)
if not form_export_schema.group_schemas:
return set()
else:
export_items = [
item for item in form_export_schema.group_schemas[0].items
if isinstance(item, ExportItem)
]
return set([
get_odata_property_from_export_item(item) for item in export_items
]) - {''}
def get_form_data_source(app, form):
xform = XForm(form.source)
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
xform.data_node.tag_xmlns,
only_process_current_builds=True,
)
meta_properties = [
_export_column_to_ucr_indicator(c) for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES
if c.label != 'form_link'
]
dynamic_properties = _get_dynamic_indicators_from_export_schema(schema)
form_name = form.default_name()
config = DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns, app.get_id),
configured_indicators=meta_properties + dynamic_properties + _get_shared_indicators(),
)
return _deduplicate_columns_if_necessary(config)
def get_export_schema(self, export_instance):
return FormExportDataSchema.generate_schema_from_builds(
self.domain, export_instance.app_id, export_instance.xmlns
)
def test(self):
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.app._id,
self.form_xmlns,
only_process_current_builds=True,
)
for group_schema in schema.group_schemas:
# group_schema is an instance of ExportGroupSchem
# group_schema.items is an array of ExportItems subclasses
path = [node.name for node in group_schema.path]
if path == []:
main_group_schema = group_schema
elif path == ['form', 'babies']:
baby_repeat_group_schema = group_schema
assertContainsExportItems(
[
# Verify that a simple form question appears in the schema
('form.how_are_you_today', 'How are you today?'),
('form.how_many_babies', 'How many babies?'),
('form.add_a_prescription', 'Add a prescription?'),
('form.voucher-name', '#form/voucher-name'),
('form.is_this_a_delivery', 'Is this a delivery?'),
('form.facility_name', '#form/facility_name'),
# Verify that the main parent case updates appear (case type "mom")
('form.case.@case_id', 'case.@case_id'),
('form.case.@date_modified', 'case.@date_modified'),
('form.case.@user_id', 'case.@user_id'),
('form.case.update.last_status', 'case.update.last_status'),
# Verify that we see case updates for save-to-case cases
# These are already in the form schema, so they get interpreted like any other question
('form.prescription.prescription.case.close',
'#form/prescription/prescription/case/close'),
('form.prescription.prescription.case.create.case_name',
'#form/prescription/prescription/case/create/case_name'),
('form.prescription.prescription.case.create.case_type',
'#form/prescription/prescription/case/create/case_type'),
('form.prescription.prescription.case.update.number_of_babies',
'#form/prescription/prescription/case/update/number_of_babies'
),
('form.prescription.prescription_name',
'#form/prescription/prescription_name'),
('form.prescription.prescription.case.index.parent',
'#form/prescription/prescription/case/index/parent'),
('form.prescription.prescription.case.@case_id',
'#form/prescription/prescription/case/@case_id'),
('form.prescription.prescription.case.@user_id',
'#form/prescription/prescription/case/@user_id'),
('form.prescription.prescription.case.@date_modified',
'#form/prescription/prescription/case/@date_modified'),
# # Verify that we see updates from subcases not in repeat groups (case type "voucher")
('form.subcase_0.case.@case_id', 'subcase_0.@case_id'),
('form.subcase_0.case.@date_modified',
'subcase_0.@date_modified'),
('form.subcase_0.case.@user_id', 'subcase_0.@user_id'),
('form.subcase_0.case.create.case_name',
'subcase_0.create.case_name'),
('form.subcase_0.case.create.case_type',
'subcase_0.create.case_type'),
('form.subcase_0.case.create.owner_id',
'subcase_0.create.owner_id'),
('form.subcase_0.case.index.parent.#text',
'subcase_0.index.#text'),
('form.subcase_0.case.index.parent.@case_type',
'subcase_0.index.@case_type'),
('form.subcase_0.case.update.how_many_babies',
'subcase_0.update.how_many_babies'),
],
main_group_schema)
# Verify that we see updates from subcases in repeat groups (case type "baby")
assertContainsExportItems([
('form.babies.case.@case_id', 'subcase_1.@case_id'),
('form.babies.case.@date_modified', 'subcase_1.@date_modified'),
('form.babies.case.@user_id', 'subcase_1.@user_id'),
('form.babies.case.create.case_name',
'subcase_1.create.case_name'),
('form.babies.case.create.case_type',
'subcase_1.create.case_type'),
('form.babies.case.create.owner_id', 'subcase_1.create.owner_id'),
('form.babies.case.update.eye_color',
'subcase_1.update.eye_color'),
('form.babies.case.update.facility_name',
'subcase_1.update.facility_name'),
('form.babies.case.index.parent.#text', 'subcase_1.index.#text'),
('form.babies.case.index.parent.@case_type',
'subcase_1.index.@case_type'),
('form.babies.eye_color', 'Eye color?'),
('form.babies.whats_the_babys_name', "What's the baby's name?"),
], baby_repeat_group_schema)
instance = FormExportInstance.generate_instance_from_schema(schema)
instance.export_format = Format.JSON
# make everything show up in the export
for table in instance.tables:
table.selected = True
for column in table.columns:
if column.item.path[
-1].name == '@case_id' and not column.item.transform:
self.assertFalse(column.is_advanced)
column.selected = True
with patch('corehq.apps.export.export.get_export_documents') as docs:
docs.return_value = self.form_es_response
export_data = get_export_json(instance)
def get_form_data(table):
headers = export_data[table]['headers']
return [
dict(zip(headers, row)) for row in export_data[table]['rows']
]
self.assertDictContainsSubset(
{
# normal form questions
"form.add_a_prescription": "yes_then_close",
"form.how_are_you_today": "fine_thanks",
"form.how_many_babies": "2",
"form.is_this_a_delivery": "yes",
"form.voucher-name": "Petunia2017-08-29",
# standard case update
"form.case.update.last_status": "fine_thanks",
"form.case.@case_id": "71626d9c-2d05-491f-81d9-becf8566618a",
"form.case.@user_id": "853a24735ba89a3019ced7e3153dc60d",
# save-to-case properties
"form.prescription.prescription.case.close": "True",
"form.prescription.prescription.case.create.case_name":
"Petunia-prescription-2017-08-29",
"form.prescription.prescription.case.create.case_type":
"prescription",
"form.prescription.prescription.case.update.number_of_babies":
"2",
"form.prescription.prescription_name":
"Petunia-prescription-2017-08-29",
"form.prescription.prescription.case.index.parent":
"71626d9c-2d05-491f-81d9-becf8566618a",
# non-repeating subcase actions
"form.subcase_0.case.@case_id":
"16954d55-a9be-40dd-98a8-dc7fae9c7ed6",
"form.subcase_0.case.@date_modified": "2017-08-29 11:19:40",
"form.subcase_0.case.@user_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.subcase_0.case.create.case_name": "Petunia2017-08-29",
"form.subcase_0.case.create.case_type": "voucher",
"form.subcase_0.case.create.owner_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.subcase_0.case.update.how_many_babies": "2",
"form.subcase_0.case.index.parent.#text":
"71626d9c-2d05-491f-81d9-becf8566618a",
"form.subcase_0.case.index.parent.@case_type": "mom",
},
get_form_data('Forms')[0])
self.assertDictEqual(
{
"number": "0.0",
"number__0": 0,
"number__1": 0,
"form.babies.eye_color": "brown",
"form.babies.whats_the_babys_name": "Bob",
"form.babies.case.@case_id":
"5539bd9d-d5d6-44c8-8f78-6915f16b6907",
"form.babies.case.@date_modified": "2017-08-29 11:19:40",
"form.babies.case.@user_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.babies.case.create.case_name": "Bob",
"form.babies.case.create.case_type": "baby",
"form.babies.case.create.owner_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.babies.case.index.parent.#text":
"71626d9c-2d05-491f-81d9-becf8566618a",
"form.babies.case.index.parent.@case_type": "mom",
"form.babies.case.update.eye_color": "brown",
"form.babies.case.update.facility_name": "test",
},
get_form_data('Repeat- babies')[0])
