def test_build_with_inferred_schema(self):
app = self.current_app
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
# Main table
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
add_inferred_export_properties(
'TestSend',
app.domain,
self.case_type,
['question2', 'new-property'],
)
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
# Main table
group_schema = schema.group_schemas[0]
# Only the new property should be added. The repeated one should be merged
self.assertEqual(len(group_schema.items), 3)
def test_build_from_saved_schema(self):
app = self.current_app
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
self.assertEqual(schema.last_app_versions[app._id], app.version)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
# After the first schema has been saved let's add a second app to process
second_build = Application.wrap(self.get_json('basic_case_application'))
second_build._id = '456'
second_build.copy_of = app.get_id
second_build.version = 6
with drop_connected_signals(app_post_save):
second_build.save()
self.addCleanup(second_build.delete)
new_schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
self.assertEqual(new_schema._id, schema._id)
self.assertEqual(new_schema.last_app_versions[app._id], app.version)
# One for case, one for case history
self.assertEqual(len(new_schema.group_schemas), 2)
def get_case_data_source(app, case_type):
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
case_type,
only_process_current_builds=True,
)
# the first two (row number and case id) are redundant/export specific,
meta_properties_to_use = MAIN_CASE_TABLE_PROPERTIES[2:]
# anything with a transform should also be removed
meta_properties_to_use = [property_def for property_def in meta_properties_to_use if property_def.item.transform is None]
meta_indicators = [_export_column_to_ucr_indicator(c) for c in meta_properties_to_use]
dynamic_indicators = _get_dynamic_indicators_from_export_schema(schema)
# filter out any duplicately defined columns from dynamic indicators
meta_column_names = set([c['column_id'] for c in meta_indicators])
dynamic_indicators = [indicator for indicator in dynamic_indicators if
indicator['column_id'] not in meta_column_names]
return DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='CommCareCase',
table_id=clean_table_name(app.domain, case_type),
display_name=case_type,
configured_filter=make_case_data_source_filter(case_type),
configured_indicators=meta_indicators + dynamic_indicators + _get_shared_indicators(),
)
def get(self, request, *args, **kwargs):
case_type = request.GET.get("export_tag").strip('"')
schema = CaseExportDataSchema.generate_schema_from_builds(self.domain, case_type, force_rebuild=True)
self.export_instance = self.export_instance_cls.generate_instance_from_schema(schema)
return super(CreateNewCustomCaseExportView, self).get(request, *args, **kwargs)
def test_basic_application_schema(self):
schema = CaseExportDataSchema.generate_schema_from_builds(self.domain, 'candy')
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(group_schema.last_occurrence, 3)
self.assertEqual(len(group_schema.items), 2)
def testCaseReferencesMakeItToCaseSchema(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
only_process_current_builds=False)
self.assertEqual(
{'save_to_case_p1', 'save_to_case_p2'},
{item.path[0].name
for item in schema.group_schemas[0].items})
def test_parent_case_table_generation_for_parent_case(self):
"""Ensures that the parent case doesn't have a parent case table"""
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
def test_parent_case_table_generation_for_parent_case(self):
"""Ensures that the parent case doesn't have a parent case table"""
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
def get_case_type_to_properties(domain):
case_type_to_properties = {}
case_types = get_case_types_for_domain_es(domain)
for case_type in case_types:
case_type_to_properties[case_type] = []
case_export_schema = CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type)
for export_group_schema in case_export_schema.group_schemas[0].items:
cleaned_case_property = export_group_schema.label.replace('_', '')
case_type_to_properties[case_type].append(cleaned_case_property)
return case_type_to_properties
def test_build_from_saved_schema(self):
app = self.current_app
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
self.assertEqual(schema.last_app_versions[app._id],
self.first_build.version)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
self.assertEqual(len(schema.group_schemas[0].items), 2)
self.assertEqual(len(schema.group_schemas[1].items), 8)
# After the first schema has been saved let's add a second app to process
second_build = Application.wrap(
self.get_json('basic_case_application'))
second_build._id = '456'
second_build.copy_of = app.get_id
second_build.version = 6
second_build.has_submissions = True
second_build.get_module(0).get_form(
0).actions.update_case.update['name'] = '/data/question2'
with drop_connected_signals(app_post_save):
second_build.save()
self.addCleanup(second_build.delete)
new_schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
self.assertEqual(new_schema._id, schema._id)
self.assertEqual(new_schema.last_app_versions[app._id],
second_build.version)
# One for case, one for case history
self.assertEqual(len(new_schema.group_schemas), 2)
self.assertEqual(len(schema.group_schemas[0].items), 2)
self.assertEqual(len(schema.group_schemas[1].items), 8)
def testCaseReferencesMakeItToCaseSchema(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
only_process_current_builds=False
)
self.assertEqual(
{'save_to_case_p1', 'save_to_case_p2'},
{item.path[0].name for item in schema.group_schemas[0].items}
)
def test_basic_delayed_schema(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
only_process_current_builds=True)
self.assertIsNone(schema.last_app_versions.get(self.current_app._id))
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
only_process_current_builds=False)
self.assertEqual(schema.last_app_versions[self.current_app._id],
self.build.version)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 3)
def test_basic_application_schema(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version)
self.assertEqual(len(group_schema.items), 2)
def test_basic_application_schema(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version)
self.assertEqual(len(group_schema.items), 2)
def test_basic_delayed_schema(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
only_process_current_builds=True
)
self.assertIsNone(schema.last_app_versions.get(self.current_app._id))
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 2)
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
only_process_current_builds=False
)
self.assertEqual(schema.last_app_versions[self.current_app._id], self.build.version)
group_schema = schema.group_schemas[0]
self.assertEqual(len(group_schema.items), 3)
def test_build_from_saved_schema(self):
app = self.current_app
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
self.assertEqual(schema.last_app_versions[app._id], self.first_build.version)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
self.assertEqual(len(schema.group_schemas[0].items), 2)
self.assertEqual(len(schema.group_schemas[1].items), 8)
# After the first schema has been saved let's add a second app to process
second_build = Application.wrap(self.get_json('basic_case_application'))
second_build._id = '456'
second_build.copy_of = app.get_id
second_build.version = 6
second_build.has_submissions = True
second_build.get_module(0).get_form(0).actions.update_case.update['name'] = '/data/question2'
with drop_connected_signals(app_post_save):
second_build.save()
self.addCleanup(second_build.delete)
new_schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
self.assertEqual(new_schema._id, schema._id)
self.assertEqual(new_schema.last_app_versions[app._id], second_build.version)
# One for case, one for case history
self.assertEqual(len(new_schema.group_schemas), 2)
self.assertEqual(len(schema.group_schemas[0].items), 2)
self.assertEqual(len(schema.group_schemas[1].items), 8)
def test_build_with_inferred_schema(self):
app = self.current_app
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
group_schema = schema.group_schemas[0]
self.assertEqual(group_schema.path, MAIN_TABLE)
self.assertTrue(group_schema.inferred)
inferred_items = [item for item in group_schema.items if item.inferred]
self.assertEqual(len(inferred_items), 2)
def get_case_properties_for_case_type(domain, case_type):
if should_use_sql_backend(domain):
from corehq.apps.export.models import CaseExportDataSchema
from corehq.apps.export.const import MAIN_TABLE
schema = CaseExportDataSchema.generate_schema_from_builds(
domain,
case_type,
)
group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE]
if group_schemas:
return sorted(set([item.path[0] for item in group_schemas[0].items]))
else:
from corehq.apps.hqcase.dbaccessors import get_case_properties
return get_case_properties(domain, case_type)
def test_build_with_inferred_schema(self):
app = self.current_app
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
self.case_type,
)
group_schema = schema.group_schemas[0]
self.assertEqual(group_schema.path, MAIN_TABLE)
self.assertTrue(group_schema.inferred)
inferred_items = [item for item in group_schema.items if item.inferred]
self.assertEqual(len(inferred_items), 2)
def get_case_properties_for_case_type(domain, case_type):
if should_use_sql_backend(domain):
from corehq.apps.export.models import CaseExportDataSchema
from corehq.apps.export.models.new import MAIN_TABLE
schema = CaseExportDataSchema.generate_schema_from_builds(
domain,
None,
case_type,
)
group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE]
if group_schemas:
return sorted(set([item.path[0].name for item in group_schemas[0].items]))
else:
from corehq.apps.hqcase.dbaccessors import get_case_properties
return get_case_properties(domain, case_type)
def get_case_type_to_properties(domain):
case_type_to_properties = defaultdict(list)
case_types = get_case_types_for_domain_es(domain)
for case_type in case_types:
if not case_type:
# TODO - understand why a case can have a blank case type and handle appropriately
continue
case_export_schema = (get_latest_case_export_schema(domain, case_type)
or
CaseExportDataSchema.generate_schema_from_builds(
domain, None, case_type))
for export_group_schema in case_export_schema.group_schemas[0].items:
cleaned_case_property = export_group_schema.label.replace('_', '')
case_type_to_properties[case_type].append(cleaned_case_property)
return dict(case_type_to_properties)
def get_case_type_to_properties(domain):
case_type_to_properties = defaultdict(list)
case_types = get_case_types_for_domain_es(domain)
for case_type in case_types:
if not case_type:
# TODO - understand why a case can have a blank case type and handle appropriately
continue
case_export_schema = (
get_latest_case_export_schema(domain, case_type)
or CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type)
)
for export_group_schema in case_export_schema.group_schemas[0].items:
cleaned_case_property = export_group_schema.label.replace('_', '')
case_type_to_properties[case_type].append(cleaned_case_property)
return dict(case_type_to_properties)
def test_multiple_app_schema_generation(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
)
self.assertEqual(
schema.last_app_versions[self.other_build.copy_of],
self.other_build.version,
)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version)
self.assertEqual(len(group_schema.items), 2)
def test_multiple_app_schema_generation(self):
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
self.case_type,
)
self.assertEqual(
schema.last_app_versions[self.other_build.copy_of],
self.other_build.version,
)
# One for case, one for case history
self.assertEqual(len(schema.group_schemas), 2)
group_schema = schema.group_schemas[0]
self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version)
self.assertEqual(len(group_schema.items), 2)
def get_case_properties_for_case_type(domain, case_type):
# todo: seems like poor boundaries for this function care about the backend
# todo: get_case_properties just always return the right answer,
# todo: possibly by moving this there.
if should_use_sql_backend(domain):
from corehq.apps.export.models import CaseExportDataSchema
from corehq.apps.export.models.new import MAIN_TABLE
schema = CaseExportDataSchema.generate_schema_from_builds(
domain,
None,
case_type,
)
group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE]
if group_schemas:
return sorted(set([item.path[0].name for item in group_schemas[0].items]))
else:
from corehq.apps.hqcase.dbaccessors import get_case_properties
return get_case_properties(domain, case_type)
def get_case_properties_for_case_type(domain, case_type):
# todo: seems like poor boundaries for this function care about the backend
# todo: get_case_properties just always return the right answer,
# todo: possibly by moving this there.
if should_use_sql_backend(domain):
from corehq.apps.export.models import CaseExportDataSchema
from corehq.apps.export.models.new import MAIN_TABLE
schema = CaseExportDataSchema.generate_schema_from_builds(
domain,
None,
case_type,
)
group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE]
if group_schemas:
return sorted(set([item.path[0].name for item in group_schemas[0].items]))
else:
from corehq.apps.hqcase.dbaccessors import get_case_properties
return get_case_properties(domain, case_type)
def test_parent_case_table_generation(self):
"""
Ensures that the child case generates a parent case table and indices
columns in main table
"""
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
'child-case',
)
# One for case, one for case history, one for parent case
self.assertEqual(len(schema.group_schemas), 3)
main_table = next(filter(lambda gs: gs.path == MAIN_TABLE, schema.group_schemas))
self.assertEqual(
len([item for item in main_table.items if item.doc_type == 'CaseIndexItem']),
1
)
self.assertEqual(
len([gs for gs in schema.group_schemas if gs.path == PARENT_CASE_TABLE]),
1
)
def test_parent_case_table_generation(self):
"""
Ensures that the child case generates a parent case table and indices
columns in main table
"""
schema = CaseExportDataSchema.generate_schema_from_builds(
self.domain,
self.current_app._id,
'child-case',
)
# One for case, one for case history, one for parent case
self.assertEqual(len(schema.group_schemas), 3)
main_table = filter(lambda gs: gs.path == MAIN_TABLE, schema.group_schemas)[0]
self.assertEqual(
len(filter(lambda item: item.doc_type == 'CaseIndexItem', main_table.items)),
1
)
self.assertEqual(
len(filter(lambda gs: gs.path == PARENT_CASE_TABLE, schema.group_schemas)),
1
)
def get_export_schema(self, export_instance):
return CaseExportDataSchema.generate_schema_from_builds(self.domain, export_instance.case_type)
