def test_build_from_saved_schema(self): app = self.current_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertEqual(len(schema.group_schemas), 1) self.assertEqual(schema.last_app_versions[app._id], self.first_build.version) # After the first schema has been saved let's add a second app to process second_build = Application.wrap(self.get_json('basic_application')) second_build._id = '456' second_build.copy_of = app.get_id second_build.version = 6 second_build.has_submissions = True second_build.save() self.addCleanup(second_build.delete) new_schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertEqual(new_schema._id, schema._id) self.assertEqual(new_schema.last_app_versions[app._id], second_build.version) self.assertEqual(len(new_schema.group_schemas), 1)
def test_rebuild_version_control(self): app = self.current_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) existing_schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertEqual(schema._id, existing_schema._id) with patch( 'corehq.apps.export.models.new.DATA_SCHEMA_VERSION', DATA_SCHEMA_VERSION + 1): rebuilt_schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertNotEqual(schema._id, rebuilt_schema._id) self.assertEqual(schema.version, DATA_SCHEMA_VERSION) self.assertEqual(rebuilt_schema.version, DATA_SCHEMA_VERSION + 1)
def test_rebuild_version_control(self): app = self.current_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) existing_schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertEqual(schema._id, existing_schema._id) with patch( 'corehq.apps.export.models.new.FORM_DATA_SCHEMA_VERSION', FORM_DATA_SCHEMA_VERSION + 1): rebuilt_schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertNotEqual(schema._id, rebuilt_schema._id) self.assertEqual(schema.version, FORM_DATA_SCHEMA_VERSION) self.assertEqual(rebuilt_schema.version, FORM_DATA_SCHEMA_VERSION + 1)
def test_build_from_saved_schema(self): app = self.current_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertEqual(len(schema.group_schemas), 1) self.assertEqual(schema.last_app_versions[app._id], app.version) # After the first schema has been saved let's add a second app to process second_build = Application.wrap(self.get_json('basic_application')) second_build._id = '456' second_build.copy_of = app.get_id second_build.version = 6 second_build.save() self.addCleanup(second_build.delete) new_schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertEqual(new_schema._id, schema._id) self.assertEqual(new_schema.last_app_versions[app._id], app.version) self.assertEqual(len(new_schema.group_schemas), 1)
def _get_merged_schema(self, form_name1, form_name2): form_xml = self.get_xml(form_name1) form_xml2 = self.get_xml(form_name2) schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1) schema2 = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml2), ['en'], self.app_id, 2) return FormExportDataSchema._merge_schemas(schema, schema2)
def test_process_app_failure(self): ''' This ensures that the schema generated will not fail if there is an error processing one of the applications. ''' with patch( 'corehq.apps.export.models.new.FormExportDataSchema._process_app_build', side_effect=Exception('boom')): FormExportDataSchema.generate_schema_from_builds( self.current_app.domain, self.current_app._id, 'my_sweet_xmlns')
def test_process_app_failure(self): ''' This ensures that the schema generated will not fail if there is an error processing one of the applications. ''' with patch( 'corehq.apps.export.models.new.FormExportDataSchema._process_app_build', side_effect=Exception('boom')): FormExportDataSchema.generate_schema_from_builds( self.current_app.domain, self.current_app._id, 'my_sweet_xmlns' )
def _get_merged_schema(self, form_name1, form_name2): form_xml = self.get_xml(form_name1) form_xml2 = self.get_xml(form_name2) schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], 1 ) schema2 = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml2), ['en'], 2 ) return FormExportDataSchema._merge_schemas(schema, schema2)
def handle(self, **options): schemas_to_rebuild = defaultdict(list) for doc_id, domain, app_id, xmlns in _latest_form_schema_ids(): schema = FormExportDataSchema.get(doc_id) group_schemas = schema.group_schemas[1:] for gs in group_schemas: if not gs.path[-1].is_repeat: schemas_to_rebuild[domain].append((app_id, xmlns)) break for domain, schema_keys in schemas_to_rebuild.items(): print("Rebuilding {} schemas for domain '{}'".format(len(schema_keys), domain)) for app_id, xmlns in schema_keys: print(" rebuilding ('{}', '{}')".format(app_id, xmlns)) FormExportDataSchema.generate_schema_from_builds(domain, app_id, xmlns, force_rebuild=True)
def test_basic_application_schema(self): app = self.current_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns') self.assertEqual(len(schema.group_schemas), 1)
def get_form_data_source(app, form): xform = XForm(form.source) schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, xform.data_node.tag_xmlns, only_process_current_builds=True, ) meta_properties = [ _export_column_to_ucr_indicator(c) for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES if c.label != 'form_link' ] dynamic_properties = _get_dynamic_indicators_from_export_schema(schema) form_name = form.default_name() config = DataSourceConfiguration( domain=app.domain, referenced_doc_type='XFormInstance', table_id=clean_table_name(app.domain, form_name), display_name=form_name, configured_filter=make_form_data_source_filter( xform.data_node.tag_xmlns, app.get_id), configured_indicators=meta_properties + dynamic_properties + _get_shared_indicators(), ) return _deduplicate_columns_if_necessary(config)
def test_xform_parsing_with_stock_questions(self): form_xml = self.get_xml('stock_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), [], ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 6) self.assertTrue(all(map(lambda item: item.doc_type == 'StockItem', group_schema.items))) for parent_attr in ['@type', '@entity-id', '@date', '@section-id']: self.assertTrue(any(map( lambda item: item.path == [ PathNode(name='form'), PathNode(name='balance:balance_one'), PathNode(name=parent_attr), ], group_schema.items, ))) for entry_attr in ['@id', '@quantity']: self.assertTrue(any(map( lambda item: item.path == [ PathNode(name='form'), PathNode(name='balance:balance_one'), PathNode(name='entry'), PathNode(name=entry_attr), ], group_schema.items, )))
def test_xform_parsing_with_repeat_group(self): form_xml = self.get_xml('repeat_group_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), [], ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) self.assertEqual(group_schema.path, MAIN_TABLE) form_items = filter(lambda item: item.tag is None, group_schema.items) self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')]) self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='zendquestion')]) group_schema = schema.group_schemas[1] self.assertEqual(len(group_schema.items), 1) self.assertEqual( group_schema.path, [PathNode(name='form'), PathNode(name='question3', is_repeat=True)] ) self.assertEqual( group_schema.items[0].path, [PathNode(name='form'), PathNode(name='question3', is_repeat=True), PathNode(name='question4')] )
def setUpClass(cls): super(TestSingleNodeRepeatConversion, cls).setUpClass() cls.schema = FormExportDataSchema( domain=cls.domain, group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[], last_occurrences={cls.app_id: 2}, ), ExportGroupSchema( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True) ], items=[ ExportItem( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='single_answer') ], label='Single Answer', last_occurrences={cls.app_id: 2}, ) ], last_occurrences={cls.app_id: 2}, ), ])
def setUpClass(cls): super(TestConversionOrdering, cls).setUpClass() cls.schema = FormExportDataSchema( domain=cls.domain, group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[PathNode(name='form'), PathNode(name='question1')], label='q1', ), ExportItem( path=[PathNode(name='form'), PathNode(name='question2')], label='q2', ), ExportItem( path=[PathNode(name='form'), PathNode(name='other')], label='other', ), ExportItem( path=[PathNode(name='form'), PathNode(name='question3')], label='q3', ), ], last_occurrences={cls.app_id: 2}, ), ] )
def test_xform_parsing_with_stock_questions(self): form_xml = self.get_xml('stock_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 6) self.assertTrue(all([item.doc_type == 'StockItem' for item in group_schema.items])) for parent_attr in ['@type', '@entity-id', '@date', '@section-id']: self.assertTrue(any(map( lambda item: item.path == [ PathNode(name='form'), PathNode(name='balance:balance_one'), PathNode(name=parent_attr), ], group_schema.items, ))) for entry_attr in ['@id', '@quantity']: self.assertTrue(any(map( lambda item: item.path == [ PathNode(name='form'), PathNode(name='balance:balance_one'), PathNode(name='entry'), PathNode(name=entry_attr), ], group_schema.items, )))
def test_xform_parsing_with_repeat_group(self): form_xml = self.get_xml('repeat_group_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) self.assertEqual(group_schema.path, MAIN_TABLE) form_items = [item for item in group_schema.items if item.tag is None] self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')]) self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='zendquestion')]) group_schema = schema.group_schemas[1] self.assertEqual(len(group_schema.items), 1) self.assertEqual( group_schema.path, [PathNode(name='form'), PathNode(name='question3', is_repeat=True)] ) self.assertEqual( group_schema.items[0].path, [PathNode(name='form'), PathNode(name='question3', is_repeat=True), PathNode(name='question4')] )
def test_build_with_advanced_app(self): app = self.advanced_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, "repeat-xmlns", ) group_schema = schema.group_schemas[1] # The repeat schema # Assert that all proper case attributes are added to advanced forms that open # cases with repeats assertContainsExportItems([ ('form.question3.question4', 'question4'), ('form.question3.case.create.case_name', 'case_open_case_0.create.case_name'), ('form.question3.case.create.case_type', 'case_open_case_0.create.case_type'), ('form.question3.case.create.owner_id', 'case_open_case_0.create.owner_id'), ('form.question3.case.index.parent.#text', 'case_open_case_0.index.#text'), ('form.question3.case.index.parent.@case_type', 'case_open_case_0.index.@case_type'), ('form.question3.case.@case_id', 'case_open_case_0.@case_id'), ('form.question3.case.@date_modified', 'case_open_case_0.@date_modified'), ('form.question3.case.@user_id', 'case_open_case_0.@user_id'), ], group_schema) path_suffixes = set( [item.path[-1].name for item in group_schema.items]) self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)), len(CASE_ATTRIBUTES))
def test_build_with_advanced_app(self): app = self.advanced_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, "repeat-xmlns", ) group_schema = schema.group_schemas[1] # The repeat schema # Assert that all proper case attributes are added to advanced forms that open # cases with repeats assertContainsExportItems( [ ('form.question3.question4', 'question4'), ('form.question3.case.create.case_name', 'case_open_case_0.create.case_name'), ('form.question3.case.create.case_type', 'case_open_case_0.create.case_type'), ('form.question3.case.create.owner_id', 'case_open_case_0.create.owner_id'), ('form.question3.case.index.parent.#text', 'case_open_case_0.index.#text'), ('form.question3.case.index.parent.@case_type', 'case_open_case_0.index.@case_type'), ('form.question3.case.@case_id', 'case_open_case_0.@case_id'), ('form.question3.case.@date_modified', 'case_open_case_0.@date_modified'), ('form.question3.case.@user_id', 'case_open_case_0.@user_id'), ], group_schema ) path_suffixes = set([item.path[-1].name for item in group_schema.items]) self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)), len(CASE_ATTRIBUTES))
def get(self, request, *args, **kwargs): app_id = request.GET.get("app_id") xmlns = request.GET.get("export_tag").strip('"') schema = FormExportDataSchema.generate_schema_from_builds(self.domain, app_id, xmlns, force_rebuild=True) self.export_instance = self.export_instance_cls.generate_instance_from_schema(schema) return super(CreateNewCustomFormExportView, self).get(request, *args, **kwargs)
def setUpClass(cls): super(TestExportDBAccessors, cls).setUpClass() cls.form_schema = FormExportDataSchema( domain=cls.domain, app_id=cls.app_id, xmlns=cls.xmlns, ) cls.form_schema_other = FormExportDataSchema( domain='other', app_id=cls.app_id, xmlns=cls.xmlns, ) cls.form_schema_before = FormExportDataSchema( domain=cls.domain, app_id=cls.app_id, xmlns=cls.xmlns, created_on=datetime.utcnow() - timedelta(1) ) cls.case_schema = CaseExportDataSchema( domain=cls.domain, case_type=cls.case_type, ) cls.case_schema_other = CaseExportDataSchema( domain=cls.domain, case_type='other', ) cls.case_schema_before = CaseExportDataSchema( domain=cls.domain, case_type=cls.case_type, created_on=datetime.utcnow() - timedelta(1) ) cls.schemas = [ cls.form_schema, cls.form_schema_before, cls.form_schema_other, cls.case_schema_before, cls.case_schema, cls.case_schema_other, ] for schema in cls.schemas: schema.save()
def test_basic_application_schema(self): app = self.current_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'my_sweet_xmlns' ) self.assertEqual(len(schema.group_schemas), 1)
def test_basic_application_schema(self): app = self.current_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, 'b68a311749a6f45bdfda015b895d607012c91613' ) self.assertEqual(len(schema.group_schemas), 1)
def test_repeat_subcases_schema_generation(self): form_xml = self.get_xml('nested_repeat_form') repeats_with_subcases = [ OpenSubCaseAction(repeat_context='/data/repeat', case_properties={ 'weight': '/data/repeat/group/weight', }), OpenSubCaseAction(repeat_context='/data/repeat/nested_repeat', case_properties={ 'age': '/data/repeat/nested_repeat/age', }), ] schema = FormExportDataSchema._generate_schema_from_repeat_subcases( XForm(form_xml), repeats_with_subcases, ['en'], self.app_id, 1, ) self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] attribute_items = filter( lambda item: item.path[-1].name in CASE_ATTRIBUTES, group_schema.items) self.assertEqual(len(attribute_items), len(CASE_ATTRIBUTES)) self.assertTrue( all( map( lambda item: item.readable_path.startswith( 'form.repeat.case'), attribute_items, ))) create_items = filter( lambda item: item.path[-1].name in CASE_CREATE_ELEMENTS, group_schema.items) self.assertEqual(len(create_items), len(CASE_CREATE_ELEMENTS)) self.assertTrue( all( map( lambda item: item.readable_path.startswith( 'form.repeat.case.create'), create_items, ))) update_items = list( set(group_schema.items) - set(create_items) - set(attribute_items)) self.assertEqual(len(update_items), 1) self.assertEqual(update_items[0].readable_path, 'form.repeat.case.update.group.weight')
def test_basic_delayed_schema(self): schema = FormExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.xmlns, only_process_current_builds=True) self.assertIsNone(schema.last_app_versions.get(self.current_app._id)) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) schema = FormExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.xmlns, only_process_current_builds=False) self.assertEqual(schema.last_app_versions[self.current_app._id], self.build.version) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 3)
def test_basic_delayed_schema(self): schema = FormExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.xmlns, only_process_current_builds=True ) self.assertIsNone(schema.last_app_versions.get(self.current_app._id)) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) schema = FormExportDataSchema.generate_schema_from_builds( self.domain, self.current_app._id, self.xmlns, only_process_current_builds=False ) self.assertEqual(schema.last_app_versions[self.current_app._id], self.build.version) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 3)
def test_build_with_advanced_app(self): app = self.advanced_app schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, "repeat-xmlns", ) group_schema = schema.group_schemas[1] # The repeat schema # Assert that all proper case attributes are added to advanced forms that open # cases with repeats path_suffixes = set(map(lambda item: item.path[-1].name, group_schema.items)) self.assertEqual(len(path_suffixes & set(CASE_ATTRIBUTES)), len(CASE_ATTRIBUTES))
def test_labels_in_xform(self): form_xml = self.get_xml('form_with_labels') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 1) self.assertEqual(group_schema.items[0].path, [PathNode(name='form'), PathNode(name='label')]) self.assertIsInstance(group_schema.items[0], LabelItem)
def setUpClass(cls): super(TestExportInstanceGenerationMultipleApps, cls).setUpClass() cls.app_id = '1234' cls.second_app_id = '5678' cls.schema = FormExportDataSchema(group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='question1') ], label='Question 1', last_occurrences={ cls.app_id: 2, cls.second_app_id: 4 }, ) ], last_occurrences={ cls.app_id: 2, cls.second_app_id: 4, }, ), ExportGroupSchema( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True) ], items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True), PathNode(name='q2') ], label='Question 2', last_occurrences={ cls.app_id: 3, }, ) ], last_occurrences={ cls.app_id: 3, }, ), ], )
def setUpClass(cls): super(TestForceConvertFormExport, cls).setUpClass() cls.project = create_domain(cls.domain) cls.project.commtrack_enabled = True cls.project.save() cls.schema = FormExportDataSchema( domain=cls.domain, app_id='123', xmlns='myxmlns', group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[], ), ], )
def _latest_form_schema_ids(): db = FormExportDataSchema.get_db() seen = set() for row in db.view('schemas_by_xmlns_or_case_type/view', reduce=False, descending=True): key_ = row['key'] doc_type = key_[1] if doc_type != 'FormExportDataSchema': continue domain, doc_type, app_id, xmlns, created_on = key_ doc_key = (domain, app_id, xmlns) if doc_key in seen: continue seen.add(doc_key) yield row['id'], domain, app_id, xmlns
def test_basic_xform_parsing(self): form_xml = self.get_xml('basic_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], 1 ) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) self.assertEqual(group_schema.items[0].path, ['data', 'question1']) self.assertEqual(group_schema.items[1].path, ['data', 'question2'])
def test_repeat_subcases_schema_generation(self): module = Module(case_type='child', _parent=Application()) form = Form().with_id(0, module) form.actions.subcases = [ OpenSubCaseAction( repeat_context='/data/repeat', case_properties={ 'weight': ConditionalCaseUpdate( question_path='/data/repeat/group/weight'), }, subcase_index=0, _nest=True).with_id(0, None), OpenSubCaseAction( repeat_context='/data/repeat', case_properties={ 'height': ConditionalCaseUpdate(question_path='/data/repeat/height'), }, subcase_index=1, _nest=True).with_id(1, None), OpenSubCaseAction( repeat_context='/data/repeat/nested_repeat', case_properties={ 'age': ConditionalCaseUpdate( question_path='/data/repeat/nested_repeat/age'), }, subcase_index=2, _nest=False).with_id(2, None), ] schema = FormExportDataSchema._add_export_items_for_cases( ExportGroupSchema(path=MAIN_TABLE), [form], ['/data/repeat', '/data/nested_repeat'], )[0] self.assertEqual(len(schema.group_schemas), len(form.actions.subcases)) for group_schema, action in zip(schema.group_schemas, form.actions.subcases): base_path = 'form.{}'.format(action.repeat_context[6:].replace( '/', '.')) if action._nest: base_path += '.{}'.format(action.form_element_name) self._check_subcase_repeat_group_schema( group_schema, list(action.case_properties), base_path)
def test_xform_parsing_with_multiple_choice(self): form_xml = self.get_xml('multiple_choice_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], 1 ) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) self.assertEqual(group_schema.items[0].path, ['data', 'question1']) self.assertEqual(group_schema.items[1].path, ['data', 'question2']) self.assertEqual(group_schema.items[1].options[0].value, 'choice1') self.assertEqual(group_schema.items[1].options[1].value, 'choice2')
def get_properties_by_xmlns(domain, app_id, xmlns): complete_xmlns = 'http://openrosa.org/formdesigner/' + xmlns form_export_schema = get_latest_form_export_schema( domain, app_id, complete_xmlns) or FormExportDataSchema.generate_schema_from_builds( domain, app_id, complete_xmlns) if not form_export_schema.group_schemas: return set() else: export_items = [ item for item in form_export_schema.group_schemas[0].items if isinstance(item, ExportItem) ] return set([ get_odata_property_from_export_item(item) for item in export_items ]) - {''}
def test_labels_in_xform(self): form_xml = self.get_xml('form_with_labels') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 1) self.assertEqual(group_schema.items[0].path, [PathNode(name='form'), PathNode(name='label')]) self.assertIsInstance(group_schema.items[0], LabelItem)
def setUpClass(cls): super(TestFormExportInstanceGeneration, cls).setUpClass() cls.app_id = '1234' cls.schema = FormExportDataSchema(group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ MultipleChoiceItem( path=[ PathNode(name='data'), PathNode(name='question1') ], label='Question 1', last_occurrences={cls.app_id: 3}, ), StockItem( path=[ PathNode(name='data'), PathNode(name='balance:question-id'), PathNode(name='@type'), ], label='Stock 1', last_occurrences={cls.app_id: 3}, ) ], last_occurrences={cls.app_id: 3}, ), ExportGroupSchema( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True) ], items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True), PathNode(name='q2') ], label='Question 2', last_occurrences={cls.app_id: 2}, ), ], last_occurrences={cls.app_id: 2}, ), ], )
def test_repeat_subcases_schema_generation(self): form_xml = self.get_xml('nested_repeat_form') repeats_with_subcases = [ OpenSubCaseAction( repeat_context='/data/repeat', case_properties={ 'weight': '/data/repeat/group/weight', } ), OpenSubCaseAction( repeat_context='/data/repeat/nested_repeat', case_properties={ 'age': '/data/repeat/nested_repeat/age', } ), ] schema = FormExportDataSchema._generate_schema_from_repeat_subcases( XForm(form_xml), repeats_with_subcases, ['en'], self.app_id, 1, ) self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] attribute_items = filter(lambda item: item.path[-1].name in CASE_ATTRIBUTES, group_schema.items) self.assertEqual(len(attribute_items), len(CASE_ATTRIBUTES)) self.assertTrue(all(map( lambda item: item.readable_path.startswith('form.repeat.case'), attribute_items, ))) create_items = filter(lambda item: item.path[-1].name in CASE_CREATE_ELEMENTS, group_schema.items) self.assertEqual(len(create_items), len(CASE_CREATE_ELEMENTS)) self.assertTrue(all(map( lambda item: item.readable_path.startswith('form.repeat.case.create'), create_items, ))) update_items = list(set(group_schema.items) - set(create_items) - set(attribute_items)) self.assertEqual(len(update_items), 1) self.assertEqual(update_items[0].readable_path, 'form.repeat.case.update.group.weight')
def test_xform_parsing_with_multiple_choice(self): form_xml = self.get_xml('multiple_choice_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), [], ['en'], self.app_id, 1) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) form_items = filter(lambda item: item.tag is None, group_schema.items) self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')]) self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='question2')]) self.assertEqual(form_items[1].options[0].value, 'choice1') self.assertEqual(form_items[1].options[1].value, 'choice2')
def test_xform_parsing_with_multiple_choice(self): form_xml = self.get_xml('multiple_choice_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2 + len(MAIN_TABLE_PROPERTIES)) form_items = filter(lambda item: item.tag is None, group_schema.items) self.assertEqual(form_items[0].path, ['form', 'question1']) self.assertEqual(form_items[1].path, ['form', 'question2']) self.assertEqual(form_items[1].options[0].value, 'choice1') self.assertEqual(form_items[1].options[1].value, 'choice2')
def test_basic_xform_parsing(self): form_xml = self.get_xml('basic_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) form_items = [item for item in group_schema.items if item.tag is None] self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')]) self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='question2')])
def test_basic_xform_parsing(self): form_xml = self.get_xml('basic_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 1) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 2) form_items = [item for item in group_schema.items if item.tag is None] self.assertEqual(form_items[0].path, [PathNode(name='form'), PathNode(name='question1')]) self.assertEqual(form_items[1].path, [PathNode(name='form'), PathNode(name='question2')])
def test_repeat_subcases_schema_generation(self): module = Module(case_type='child', _parent=Application()) form = Form().with_id(0, module) form.actions.subcases = [ OpenSubCaseAction( repeat_context='/data/repeat', case_properties={ 'weight': '/data/repeat/group/weight', }, subcase_index=0, _nest=True ).with_id(0, None), OpenSubCaseAction( repeat_context='/data/repeat', case_properties={ 'height': '/data/repeat/height', }, subcase_index=1, _nest=True ).with_id(1, None), OpenSubCaseAction( repeat_context='/data/repeat/nested_repeat', case_properties={ 'age': '/data/repeat/nested_repeat/age', }, subcase_index=2, _nest=False ).with_id(2, None), ] schema = FormExportDataSchema._add_export_items_for_cases( ExportGroupSchema(path=MAIN_TABLE), [form], ['/data/repeat', '/data/nested_repeat'], )[0] self.assertEqual(len(schema.group_schemas), len(form.actions.subcases)) for group_schema, action in zip(schema.group_schemas, form.actions.subcases): base_path = 'form.{}'.format(action.repeat_context[6:].replace('/', '.')) if action._nest: base_path += '.{}'.format(action.form_element_name) self._check_subcase_repeat_group_schema(group_schema, list(action.case_properties), base_path)
def setUpClass(cls): super(TestConvertStockFormExport, cls).setUpClass() cls.schema = FormExportDataSchema( domain=cls.domain, group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ StockItem( path=[ PathNode(name='form'), PathNode(name='transfer:questionid'), PathNode(name='entry'), PathNode(name='@id'), ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ], last_occurrences={cls.app_id: 2}, ), ExportGroupSchema( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True) ], items=[ StockItem( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='transfer:questionid'), PathNode(name='entry'), PathNode(name='@id'), ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ], last_occurrences={cls.app_id: 2}, ), ])
def setUpClass(cls): super(TestConvertMissingAppID, cls).setUpClass() factory = AppFactory(cls.domain) m1, m1f1 = factory.new_basic_module('open_case', 'house') m1f1.xmlns = 'missing-xmlns' cls.app = factory.app cls.app.save() cls.project = create_domain(cls.domain) cls.schema = FormExportDataSchema( domain=cls.domain, app_id='123', xmlns='myxmlns', group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[], ), ], )
def test_xform_parsing_with_repeat_group(self): form_xml = self.get_xml('repeat_group_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], 1 ) self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 1) self.assertEqual(group_schema.path, None) self.assertEqual(group_schema.items[0].path, ['data', 'question1']) group_schema = schema.group_schemas[1] self.assertEqual(len(group_schema.items), 1) self.assertEqual(group_schema.path, ['data', 'question3']) self.assertEqual(group_schema.items[0].path, ['data', 'question3', 'question4'])
def test_xform_parsing_with_repeat_group(self): form_xml = self.get_xml('repeat_group_form') schema = FormExportDataSchema._generate_schema_from_xform( XForm(form_xml), ['en'], self.app_id, 1 ) self.assertEqual(len(schema.group_schemas), 2) group_schema = schema.group_schemas[0] self.assertEqual(len(group_schema.items), 1 + len(MAIN_TABLE_PROPERTIES)) self.assertEqual(group_schema.path, MAIN_TABLE) form_items = filter(lambda item: item.tag is None, group_schema.items) self.assertEqual(form_items[0].path, ['form', 'question1']) group_schema = schema.group_schemas[1] self.assertEqual(len(group_schema.items), 1) self.assertEqual(group_schema.path, ['form', 'question3']) self.assertEqual(group_schema.items[0].path, ['form', 'question3', 'question4'])
def get_form_data_source(app, form): xform = XForm(form.source) schema = FormExportDataSchema.generate_schema_from_builds( app.domain, app._id, xform.data_node.tag_xmlns, only_process_current_builds=True, ) meta_properties = [ _export_column_to_ucr_indicator(c) for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES if c.label != 'form_link' ] dynamic_properties = _get_dynamic_indicators_from_export_schema(schema) form_name = form.default_name() config = DataSourceConfiguration( domain=app.domain, referenced_doc_type='XFormInstance', table_id=clean_table_name(app.domain, form_name), display_name=form_name, configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns, app.get_id), configured_indicators=meta_properties + dynamic_properties + _get_shared_indicators(), ) return _deduplicate_columns_if_necessary(config)
def test(self): schema = FormExportDataSchema.generate_schema_from_builds( self.domain, self.app._id, self.form_xmlns, only_process_current_builds=True, ) for group_schema in schema.group_schemas: # group_schema is an instance of ExportGroupSchem # group_schema.items is an array of ExportItems subclasses path = [node.name for node in group_schema.path] if path == []: main_group_schema = group_schema elif path == ['form', 'babies']: baby_repeat_group_schema = group_schema assertContainsExportItems( [ # Verify that a simple form question appears in the schema ('form.how_are_you_today', 'How are you today?'), ('form.how_many_babies', 'How many babies?'), ('form.add_a_prescription', 'Add a prescription?'), ('form.voucher-name', '#form/voucher-name'), ('form.is_this_a_delivery', 'Is this a delivery?'), ('form.facility_name', '#form/facility_name'), # Verify that the main parent case updates appear (case type "mom") ('form.case.@case_id', 'case.@case_id'), ('form.case.@date_modified', 'case.@date_modified'), ('form.case.@user_id', 'case.@user_id'), ('form.case.update.last_status', 'case.update.last_status'), # Verify that we see case updates for save-to-case cases # These are already in the form schema, so they get interpreted like any other question ('form.prescription.prescription.case.close', '#form/prescription/prescription/case/close'), ('form.prescription.prescription.case.create.case_name', '#form/prescription/prescription/case/create/case_name'), ('form.prescription.prescription.case.create.case_type', '#form/prescription/prescription/case/create/case_type'), ('form.prescription.prescription.case.update.number_of_babies', '#form/prescription/prescription/case/update/number_of_babies' ), ('form.prescription.prescription_name', '#form/prescription/prescription_name'), ('form.prescription.prescription.case.index.parent', '#form/prescription/prescription/case/index/parent'), ('form.prescription.prescription.case.@case_id', '#form/prescription/prescription/case/@case_id'), ('form.prescription.prescription.case.@user_id', '#form/prescription/prescription/case/@user_id'), ('form.prescription.prescription.case.@date_modified', '#form/prescription/prescription/case/@date_modified'), # # Verify that we see updates from subcases not in repeat groups (case type "voucher") ('form.subcase_0.case.@case_id', 'subcase_0.@case_id'), ('form.subcase_0.case.@date_modified', 'subcase_0.@date_modified'), ('form.subcase_0.case.@user_id', 'subcase_0.@user_id'), ('form.subcase_0.case.create.case_name', 'subcase_0.create.case_name'), ('form.subcase_0.case.create.case_type', 'subcase_0.create.case_type'), ('form.subcase_0.case.create.owner_id', 'subcase_0.create.owner_id'), ('form.subcase_0.case.index.parent.#text', 'subcase_0.index.#text'), ('form.subcase_0.case.index.parent.@case_type', 'subcase_0.index.@case_type'), ('form.subcase_0.case.update.how_many_babies', 'subcase_0.update.how_many_babies'), ], main_group_schema) # Verify that we see updates from subcases in repeat groups (case type "baby") assertContainsExportItems([ ('form.babies.case.@case_id', 'subcase_1.@case_id'), ('form.babies.case.@date_modified', 'subcase_1.@date_modified'), ('form.babies.case.@user_id', 'subcase_1.@user_id'), ('form.babies.case.create.case_name', 'subcase_1.create.case_name'), ('form.babies.case.create.case_type', 'subcase_1.create.case_type'), ('form.babies.case.create.owner_id', 'subcase_1.create.owner_id'), ('form.babies.case.update.eye_color', 'subcase_1.update.eye_color'), ('form.babies.case.update.facility_name', 'subcase_1.update.facility_name'), ('form.babies.case.index.parent.#text', 'subcase_1.index.#text'), ('form.babies.case.index.parent.@case_type', 'subcase_1.index.@case_type'), ('form.babies.eye_color', 'Eye color?'), ('form.babies.whats_the_babys_name', "What's the baby's name?"), ], baby_repeat_group_schema) instance = FormExportInstance.generate_instance_from_schema(schema) instance.export_format = Format.JSON # make everything show up in the export for table in instance.tables: table.selected = True for column in table.columns: if column.item.path[ -1].name == '@case_id' and not column.item.transform: self.assertFalse(column.is_advanced) column.selected = True with patch('corehq.apps.export.export.get_export_documents') as docs: docs.return_value = self.form_es_response export_data = get_export_json(instance) def get_form_data(table): headers = export_data[table]['headers'] return [ dict(zip(headers, row)) for row in export_data[table]['rows'] ] self.assertDictContainsSubset( { # normal form questions "form.add_a_prescription": "yes_then_close", "form.how_are_you_today": "fine_thanks", "form.how_many_babies": "2", "form.is_this_a_delivery": "yes", "form.voucher-name": "Petunia2017-08-29", # standard case update "form.case.update.last_status": "fine_thanks", "form.case.@case_id": "71626d9c-2d05-491f-81d9-becf8566618a", "form.case.@user_id": "853a24735ba89a3019ced7e3153dc60d", # save-to-case properties "form.prescription.prescription.case.close": "True", "form.prescription.prescription.case.create.case_name": "Petunia-prescription-2017-08-29", "form.prescription.prescription.case.create.case_type": "prescription", "form.prescription.prescription.case.update.number_of_babies": "2", "form.prescription.prescription_name": "Petunia-prescription-2017-08-29", "form.prescription.prescription.case.index.parent": "71626d9c-2d05-491f-81d9-becf8566618a", # non-repeating subcase actions "form.subcase_0.case.@case_id": "16954d55-a9be-40dd-98a8-dc7fae9c7ed6", "form.subcase_0.case.@date_modified": "2017-08-29 11:19:40", "form.subcase_0.case.@user_id": "853a24735ba89a3019ced7e3153dc60d", "form.subcase_0.case.create.case_name": "Petunia2017-08-29", "form.subcase_0.case.create.case_type": "voucher", "form.subcase_0.case.create.owner_id": "853a24735ba89a3019ced7e3153dc60d", "form.subcase_0.case.update.how_many_babies": "2", "form.subcase_0.case.index.parent.#text": "71626d9c-2d05-491f-81d9-becf8566618a", "form.subcase_0.case.index.parent.@case_type": "mom", }, get_form_data('Forms')[0]) self.assertDictEqual( { "number": "0.0", "number__0": 0, "number__1": 0, "form.babies.eye_color": "brown", "form.babies.whats_the_babys_name": "Bob", "form.babies.case.@case_id": "5539bd9d-d5d6-44c8-8f78-6915f16b6907", "form.babies.case.@date_modified": "2017-08-29 11:19:40", "form.babies.case.@user_id": "853a24735ba89a3019ced7e3153dc60d", "form.babies.case.create.case_name": "Bob", "form.babies.case.create.case_type": "baby", "form.babies.case.create.owner_id": "853a24735ba89a3019ced7e3153dc60d", "form.babies.case.index.parent.#text": "71626d9c-2d05-491f-81d9-becf8566618a", "form.babies.case.index.parent.@case_type": "mom", "form.babies.case.update.eye_color": "brown", "form.babies.case.update.facility_name": "test", }, get_form_data('Repeat- babies')[0])
def setUpClass(cls): super(TestConvertSavedExportSchemaToFormExportInstance, cls).setUpClass() cls.schema = FormExportDataSchema( domain=cls.domain, group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[ PathNode(name='form'), PathNode(name='question1') ], label='Question 1 Not updated', last_occurrences={cls.app_id: 3}, ), ExportItem( path=[ PathNode(name='form'), PathNode(name='deid_id') ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ExportItem( path=[ PathNode(name='form'), PathNode(name='deid_date') ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ], last_occurrences={cls.app_id: 3}, ), ExportGroupSchema( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True) ], items=[ ExportItem( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='question2') ], label='Question 2', last_occurrences={cls.app_id: 2}, ) ], last_occurrences={cls.app_id: 2}, ), ExportGroupSchema( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='repeat_nested', is_repeat=True), ], items=[ ExportItem( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='repeat_nested', is_repeat=True), PathNode(name='nested'), ], label='Nested Repeat', last_occurrences={cls.app_id: 2}, ) ], last_occurrences={cls.app_id: 2}, ), ], )
def setUpClass(cls): super(TestExportInstanceFromSavedInstance, cls).setUpClass() cls.app_id = '1234' cls.schema = FormExportDataSchema(group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='question1') ], label='Question 1', last_occurrences={ cls.app_id: 3, }, ), ExportItem( path=[ PathNode(name='data'), PathNode(name='@case_id') ], label='@case_id', tag=PROPERTY_TAG_CASE, last_occurrences={ cls.app_id: 3, }, ) ], last_occurrences={ cls.app_id: 3, }, ), ], ) cls.new_schema = FormExportDataSchema(group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='question1') ], label='Question 1', last_occurrences={ cls.app_id: 3, }, ), ExportItem( path=[ PathNode(name='data'), PathNode(name='@case_id') ], label='@case_id', tag=PROPERTY_TAG_CASE, last_occurrences={ cls.app_id: 3, }, ), ExportItem( path=[ PathNode(name='data'), PathNode(name='question3') ], label='Question 3', last_occurrences={ cls.app_id: 3, }, ) ], last_occurrences={ cls.app_id: 3, }, ), ExportGroupSchema( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True) ], items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True), PathNode(name='q2') ], label='Question 2', last_occurrences={ cls.app_id: 3, }, ) ], last_occurrences={ cls.app_id: 3, }, ), ], )
def get_export_schema(self, export_instance): return FormExportDataSchema.generate_schema_from_builds( self.domain, export_instance.app_id, export_instance.xmlns )