def test_build_with_inferred_schema(self): app = self.current_app schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) # Main table group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) add_inferred_export_properties( 'TestSend', app.domain, self.case_type, ['question2', 'new-property'], ) schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) # Main table group_schema = schema.group_schemas[0] # Only the new property should be added. The repeated one should be merged self.assertEqual(len(group_schema.items), 3)
def test_build_from_saved_schema(self): app = self.current_app schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) self.assertEqual(schema.last_app_versions[app._id], app.version) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2) # After the first schema has been saved let's add a second app to process second_build = Application.wrap(self.get_json('basic_case_application')) second_build._id = '456' second_build.copy_of = app.get_id second_build.version = 6 with drop_connected_signals(app_post_save): second_build.save() self.addCleanup(second_build.delete) new_schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) self.assertEqual(new_schema._id, schema._id) self.assertEqual(new_schema.last_app_versions[app._id], app.version) # One for case, one for case history self.assertEqual(len(new_schema.group_schemas), 2)
def get_case_data_source(app, case_type): schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, case_type, only_process_current_builds=True, ) # the first two (row number and case id) are redundant/export specific, meta_properties_to_use = MAIN_CASE_TABLE_PROPERTIES[2:] # anything with a transform should also be removed meta_properties_to_use = [property_def for property_def in meta_properties_to_use if property_def.item.transform is None] meta_indicators = [_export_column_to_ucr_indicator(c) for c in meta_properties_to_use] dynamic_indicators = _get_dynamic_indicators_from_export_schema(schema) # filter out any duplicately defined columns from dynamic indicators meta_column_names = set([c['column_id'] for c in meta_indicators]) dynamic_indicators = [indicator for indicator in dynamic_indicators if indicator['column_id'] not in meta_column_names] return DataSourceConfiguration( domain=app.domain, referenced_doc_type='CommCareCase', table_id=clean_table_name(app.domain, case_type), display_name=case_type, configured_filter=make_case_data_source_filter(case_type), configured_indicators=meta_indicators + dynamic_indicators + _get_shared_indicators(), )
def get(self, request, *args, **kwargs): case_type = request.GET.get("export_tag").strip('"') schema = CaseExportDataSchema.generate_schema_from_builds(self.domain, case_type, force_rebuild=True) self.export_instance = self.export_instance_cls.generate_instance_from_schema(schema) return super(CreateNewCustomCaseExportView, self).get(request, *args, **kwargs)
def test_basic_application_schema(self): schema = CaseExportDataSchema.generate_schema_from_builds(self.domain, 'candy') self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(group_schema.last_occurrence, 3) self.assertEqual(len(group_schema.items), 2)
def testCaseReferencesMakeItToCaseSchema(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, only_process_current_builds=False) self.assertEqual( {'save_to_case_p1', 'save_to_case_p2'}, {item.path[0].name for item in schema.group_schemas[0].items})
def test_parent_case_table_generation_for_parent_case(self): """Ensures that the parent case doesn't have a parent case table""" schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, ) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2)
def test_parent_case_table_generation_for_parent_case(self): """Ensures that the parent case doesn't have a parent case table""" schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, ) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2)
def get_case_type_to_properties(domain): case_type_to_properties = {} case_types = get_case_types_for_domain_es(domain) for case_type in case_types: case_type_to_properties[case_type] = [] case_export_schema = CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type) for export_group_schema in case_export_schema.group_schemas[0].items: cleaned_case_property = export_group_schema.label.replace('_', '') case_type_to_properties[case_type].append(cleaned_case_property) return case_type_to_properties
def test_build_from_saved_schema(self): app = self.current_app schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) self.assertEqual(schema.last_app_versions[app._id], self.first_build.version) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2) self.assertEqual(len(schema.group_schemas[0].items), 2) self.assertEqual(len(schema.group_schemas[1].items), 8) # After the first schema has been saved let's add a second app to process second_build = Application.wrap( self.get_json('basic_case_application')) second_build._id = '456' second_build.copy_of = app.get_id second_build.version = 6 second_build.has_submissions = True second_build.get_module(0).get_form( 0).actions.update_case.update['name'] = '/data/question2' with drop_connected_signals(app_post_save): second_build.save() self.addCleanup(second_build.delete) new_schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) self.assertEqual(new_schema._id, schema._id) self.assertEqual(new_schema.last_app_versions[app._id], second_build.version) # One for case, one for case history self.assertEqual(len(new_schema.group_schemas), 2) self.assertEqual(len(schema.group_schemas[0].items), 2) self.assertEqual(len(schema.group_schemas[1].items), 8)
def testCaseReferencesMakeItToCaseSchema(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, only_process_current_builds=False ) self.assertEqual( {'save_to_case_p1', 'save_to_case_p2'}, {item.path[0].name for item in schema.group_schemas[0].items} )
def test_basic_delayed_schema(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, only_process_current_builds=True) self.assertIsNone(schema.last_app_versions.get(self.current_app._id)) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, only_process_current_builds=False) self.assertEqual(schema.last_app_versions[self.current_app._id], self.build.version) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 3)
def test_basic_application_schema(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, ) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version) self.assertEqual(len(group_schema.items), 2)
def test_basic_application_schema(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, ) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version) self.assertEqual(len(group_schema.items), 2)
def test_basic_delayed_schema(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, only_process_current_builds=True ) self.assertIsNone(schema.last_app_versions.get(self.current_app._id)) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, only_process_current_builds=False ) self.assertEqual(schema.last_app_versions[self.current_app._id], self.build.version) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 3)
def test_build_from_saved_schema(self): app = self.current_app schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) self.assertEqual(schema.last_app_versions[app._id], self.first_build.version) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2) self.assertEqual(len(schema.group_schemas[0].items), 2) self.assertEqual(len(schema.group_schemas[1].items), 8) # After the first schema has been saved let's add a second app to process second_build = Application.wrap(self.get_json('basic_case_application')) second_build._id = '456' second_build.copy_of = app.get_id second_build.version = 6 second_build.has_submissions = True second_build.get_module(0).get_form(0).actions.update_case.update['name'] = '/data/question2' with drop_connected_signals(app_post_save): second_build.save() self.addCleanup(second_build.delete) new_schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) self.assertEqual(new_schema._id, schema._id) self.assertEqual(new_schema.last_app_versions[app._id], second_build.version) # One for case, one for case history self.assertEqual(len(new_schema.group_schemas), 2) self.assertEqual(len(schema.group_schemas[0].items), 2) self.assertEqual(len(schema.group_schemas[1].items), 8)
def test_build_with_inferred_schema(self): app = self.current_app schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) group_schema = schema.group_schemas[0] self.assertEqual(group_schema.path, MAIN_TABLE) self.assertTrue(group_schema.inferred) inferred_items = [item for item in group_schema.items if item.inferred] self.assertEqual(len(inferred_items), 2)
def get_case_properties_for_case_type(domain, case_type): if should_use_sql_backend(domain): from corehq.apps.export.models import CaseExportDataSchema from corehq.apps.export.const import MAIN_TABLE schema = CaseExportDataSchema.generate_schema_from_builds( domain, case_type, ) group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE] if group_schemas: return sorted(set([item.path[0] for item in group_schemas[0].items])) else: from corehq.apps.hqcase.dbaccessors import get_case_properties return get_case_properties(domain, case_type)
def test_build_with_inferred_schema(self): app = self.current_app schema = CaseExportDataSchema.generate_schema_from_builds( app.domain, app._id, self.case_type, ) group_schema = schema.group_schemas[0] self.assertEqual(group_schema.path, MAIN_TABLE) self.assertTrue(group_schema.inferred) inferred_items = [item for item in group_schema.items if item.inferred] self.assertEqual(len(inferred_items), 2)
def get_case_properties_for_case_type(domain, case_type): if should_use_sql_backend(domain): from corehq.apps.export.models import CaseExportDataSchema from corehq.apps.export.models.new import MAIN_TABLE schema = CaseExportDataSchema.generate_schema_from_builds( domain, None, case_type, ) group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE] if group_schemas: return sorted(set([item.path[0].name for item in group_schemas[0].items])) else: from corehq.apps.hqcase.dbaccessors import get_case_properties return get_case_properties(domain, case_type)
def get_case_type_to_properties(domain): case_type_to_properties = defaultdict(list) case_types = get_case_types_for_domain_es(domain) for case_type in case_types: if not case_type: # TODO - understand why a case can have a blank case type and handle appropriately continue case_export_schema = (get_latest_case_export_schema(domain, case_type) or CaseExportDataSchema.generate_schema_from_builds( domain, None, case_type)) for export_group_schema in case_export_schema.group_schemas[0].items: cleaned_case_property = export_group_schema.label.replace('_', '') case_type_to_properties[case_type].append(cleaned_case_property) return dict(case_type_to_properties)
def get_case_type_to_properties(domain): case_type_to_properties = defaultdict(list) case_types = get_case_types_for_domain_es(domain) for case_type in case_types: if not case_type: # TODO - understand why a case can have a blank case type and handle appropriately continue case_export_schema = ( get_latest_case_export_schema(domain, case_type) or CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type) ) for export_group_schema in case_export_schema.group_schemas[0].items: cleaned_case_property = export_group_schema.label.replace('_', '') case_type_to_properties[case_type].append(cleaned_case_property) return dict(case_type_to_properties)
def test_multiple_app_schema_generation(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, ) self.assertEqual( schema.last_app_versions[self.other_build.copy_of], self.other_build.version, ) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version) self.assertEqual(len(group_schema.items), 2)
def test_multiple_app_schema_generation(self): schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.case_type, ) self.assertEqual( schema.last_app_versions[self.other_build.copy_of], self.other_build.version, ) # One for case, one for case history self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(group_schema.last_occurrences[self.current_app._id], self.current_app.version) self.assertEqual(len(group_schema.items), 2)
def get_case_properties_for_case_type(domain, case_type): # todo: seems like poor boundaries for this function care about the backend # todo: get_case_properties just always return the right answer, # todo: possibly by moving this there. if should_use_sql_backend(domain): from corehq.apps.export.models import CaseExportDataSchema from corehq.apps.export.models.new import MAIN_TABLE schema = CaseExportDataSchema.generate_schema_from_builds( domain, None, case_type, ) group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE] if group_schemas: return sorted(set([item.path[0].name for item in group_schemas[0].items])) else: from corehq.apps.hqcase.dbaccessors import get_case_properties return get_case_properties(domain, case_type)
def get_case_properties_for_case_type(domain, case_type): # todo: seems like poor boundaries for this function care about the backend # todo: get_case_properties just always return the right answer, # todo: possibly by moving this there. if should_use_sql_backend(domain): from corehq.apps.export.models import CaseExportDataSchema from corehq.apps.export.models.new import MAIN_TABLE schema = CaseExportDataSchema.generate_schema_from_builds( domain, None, case_type, ) group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE] if group_schemas: return sorted(set([item.path[0].name for item in group_schemas[0].items])) else: from corehq.apps.hqcase.dbaccessors import get_case_properties return get_case_properties(domain, case_type)
def test_parent_case_table_generation(self): """ Ensures that the child case generates a parent case table and indices columns in main table """ schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, 'child-case', ) # One for case, one for case history, one for parent case self.assertEqual(len(schema.group_schemas), 3) main_table = next(filter(lambda gs: gs.path == MAIN_TABLE, schema.group_schemas)) self.assertEqual( len([item for item in main_table.items if item.doc_type == 'CaseIndexItem']), 1 ) self.assertEqual( len([gs for gs in schema.group_schemas if gs.path == PARENT_CASE_TABLE]), 1 )
def test_parent_case_table_generation(self): """ Ensures that the child case generates a parent case table and indices columns in main table """ schema = CaseExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, 'child-case', ) # One for case, one for case history, one for parent case self.assertEqual(len(schema.group_schemas), 3) main_table = filter(lambda gs: gs.path == MAIN_TABLE, schema.group_schemas)[0] self.assertEqual( len(filter(lambda item: item.doc_type == 'CaseIndexItem', main_table.items)), 1 ) self.assertEqual( len(filter(lambda gs: gs.path == PARENT_CASE_TABLE, schema.group_schemas)), 1 )
def get_export_schema(self, export_instance): return CaseExportDataSchema.generate_schema_from_builds(self.domain, export_instance.case_type)