def test_revert_new_exports_restore_old(self): saved_export_schema = SavedExportSchema(index=['my-domain', 'xmlns']) saved_export_schema.doc_type += DELETED_SUFFIX saved_export_schema.save() self.new_exports[ 0].legacy_saved_export_schema_id = saved_export_schema._id reverted = revert_new_exports(self.new_exports) self.assertEqual(len(reverted), 1) self.assertFalse(reverted[0].doc_type.endswith(DELETED_SUFFIX)) saved_export_schema.delete() class TestConvertIndexToPath(SimpleTestCase): """Test the conversion of old style index to new style path""" @generate_cases( [ ('form.question1', [PathNode(name='form'), PathNode(name='question1')]), ('#', MAIN_TABLE), ('#.form.question1.#', [PathNode(name='form'), PathNode(name='question1', is_repeat=True)]), # Repeat group ], TestConvertIndexToPath) def test_convert_index_to_path_nodes(self, index, path): self.assertEqual(_convert_index_to_path_nodes(index), path)
from corehq.apps.export.models import ( ExportColumn, ExportItem, PathNode, StockExportColumn, RowNumberColumn, SplitGPSExportColumn, GeopointItem, ) # System properties to be displayed above the form questions TOP_MAIN_FORM_TABLE_PROPERTIES = [ RowNumberColumn( tags=[PROPERTY_TAG_ROW], label="number", item=ExportItem(path=[PathNode(name='number')]), selected=True, ), ExportColumn( tags=[PROPERTY_TAG_INFO], label="formid", item=ExportItem(path=[ PathNode(name='form'), PathNode(name='meta'), PathNode(name='instanceID') ]), help_text=_('Unique identifier of the form submission'), selected=True, ) ]
def setUpClass(cls): super(TestExportInstanceFromSavedInstance, cls).setUpClass() cls.app_id = '1234' cls.schema = FormExportDataSchema(group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='question1') ], label='Question 1', last_occurrences={ cls.app_id: 3, }, ), ExportItem( path=[ PathNode(name='data'), PathNode(name='@case_id') ], label='@case_id', tag=PROPERTY_TAG_CASE, last_occurrences={ cls.app_id: 3, }, ) ], last_occurrences={ cls.app_id: 3, }, ), ], ) cls.new_schema = FormExportDataSchema(group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='question1') ], label='Question 1', last_occurrences={ cls.app_id: 3, }, ), ExportItem( path=[ PathNode(name='data'), PathNode(name='@case_id') ], label='@case_id', tag=PROPERTY_TAG_CASE, last_occurrences={ cls.app_id: 3, }, ), ExportItem( path=[ PathNode(name='data'), PathNode(name='question3') ], label='Question 3', last_occurrences={ cls.app_id: 3, }, ) ], last_occurrences={ cls.app_id: 3, }, ), ExportGroupSchema( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True) ], items=[ ExportItem( path=[ PathNode(name='data'), PathNode(name='repeat', is_repeat=True), PathNode(name='q2') ], label='Question 2', last_occurrences={ cls.app_id: 3, }, ) ], last_occurrences={ cls.app_id: 3, }, ), ], )
def test_get_column(self): table_configuration = TableConfiguration( path=[PathNode(name='form', is_repeat=False), PathNode(name="repeat1", is_repeat=True)], columns=[ ExportColumn( item=ScalarItem( path=[ PathNode(name='form'), PathNode(name='repeat1', is_repeat=True), PathNode(name='q1') ], ) ), ExportColumn( item=ScalarItem( path=[ PathNode(name="form"), PathNode(name="user_id"), ], transform=USERNAME_TRANSFORM ) ), ExportColumn( item=ScalarItem( path=[ PathNode(name='form'), PathNode(name='repeat1', is_repeat=True), PathNode(name='q2') ], ) ), ] ) index, column = table_configuration.get_column( [ PathNode(name='form'), PathNode(name='repeat1', is_repeat=True), PathNode(name='q1') ], 'ScalarItem', None, ) self.assertEqual( column.item.path, [ PathNode(name='form'), PathNode(name='repeat1', is_repeat=True), PathNode(name='q1') ] ) self.assertEqual(index, 0) index, column = table_configuration.get_column( [ PathNode(name='form'), PathNode(name='repeat1', is_repeat=True), PathNode(name='DoesNotExist') ], 'ScalarItem', None, ) self.assertIsNone(column) # Verify that get_column ignores deid transforms index, column = table_configuration.get_column( [PathNode(name="form"), PathNode(name="user_id")], 'ScalarItem', USERNAME_TRANSFORM ) self.assertIsNotNone(column) self.assertEqual(index, 1)
PathNode, StockExportColumn, RowNumberColumn, SplitGPSExportColumn, GeopointItem, ScalarItem, ) # System properties to be displayed above the form questions from corehq.apps.userreports.datatypes import DATA_TYPE_DATETIME, DATA_TYPE_STRING TOP_MAIN_FORM_TABLE_PROPERTIES = [ RowNumberColumn( tags=[PROPERTY_TAG_ROW], label="number", item=ExportItem(path=[PathNode(name='number')]), selected=True, ), ExportColumn( tags=[PROPERTY_TAG_INFO], label="formid", item=ExportItem(path=[ PathNode(name='form'), PathNode(name='meta'), PathNode(name='instanceID') ]), help_text=_('Unique identifier of the form submission'), selected=True, ) ]
def test_empty_string(self): doc = {'external_blobs': {}, 'photo': ''} result = self.column.get_value('my-domain', '1234', doc, [PathNode(name='form')]) self.assertEqual(result, '')
def test_simple_bulk_export(self): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( selected=True, label="My table", path=MAIN_TABLE, columns=[ ExportColumn( label="Foo column", item=ExportItem( path=[PathNode(name="foo")] ), selected=True, ), ] )] ), CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=MAIN_TABLE, columns=[ ExportColumn( label="Bar column", item=ExportItem( path=[PathNode(name="bar")] ), selected=True, ) ] )] ), ], [] # No filters ) expected = { 'Export1-My table': { "A1": "Foo column", "A2": "apple", "A3": "apple", "A4": "apple", }, "Export2-My table": { "A1": "Bar column", "A2": "banana", "A3": "banana", "A4": "banana", }, } with export_file as export: wb = load_workbook(export) self.assertEqual(wb.get_sheet_names(), ["Export1-My table", "Export2-My table"]) for sheet in expected.keys(): for cell in expected[sheet].keys(): self.assertEqual( wb[sheet][cell].value, expected[sheet][cell], 'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format( sheet, cell, expected[sheet][cell], wb[sheet][cell].value ) )
def test_multiple_write_export_instance_calls(self, export_save): """ Confirm that calling _write_export_instance() multiple times (as part of a bulk export) works as expected. """ export_instances = [ FormExportInstance(tables=[ TableConfiguration(label="My table", selected=True, path=[], columns=[ ExportColumn( label="Q3", item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q3') ], ), selected=True, ), ]), ]), FormExportInstance(tables=[ TableConfiguration(label="My other table", selected=True, path=[ PathNode(name="form", is_repeat=False), PathNode(name="q2", is_repeat=False) ], columns=[ ExportColumn( label="Q4", item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4') ], ), selected=True, ), ]) ]), FormExportInstance(tables=[ TableConfiguration(label="My other table", selected=True, path=[ PathNode(name="form", is_repeat=False), PathNode(name="q2", is_repeat=False) ], columns=[ ExportColumn( label="Q4", item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4') ], ), selected=True, ), ]) ]) ] with TransientTempfile() as temp_path: writer = _ExportWriter(get_writer(Format.JSON), temp_path) with writer.open(export_instances): write_export_instance(writer, export_instances[0], self.docs) write_export_instance(writer, export_instances[1], self.docs) write_export_instance(writer, export_instances[2], self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { 'My table': { 'headers': ['Q3'], 'rows': [['baz'], ['bop']], }, 'Export2-My other table': { 'headers': ['Q4'], 'rows': [['bar'], ['boop']], }, 'Export3-My other table': { 'headers': ['Q4'], 'rows': [['bar'], ['boop']], }, }) self.assertTrue(export_save.called)
def test_form_stock_columns(self): """Ensure that we can export stock properties in a form export""" docs = [{ '_id': 'simone-biles', 'domain': DOMAIN, 'form': { 'balance': [ { '@type': 'question-id', 'entry': { '@quantity': '2', } }, { '@type': 'other-question-id', 'entry': { '@quantity': '3', } }] }, }, { '_id': 'sam-mikulak', 'domain': DOMAIN, 'form': { 'balance': { '@type': 'question-id', 'entry': { '@quantity': '2', } }, }, }, { '_id': 'kerri-walsh', 'domain': DOMAIN, 'form': { 'balance': { '@type': 'other-question-id', 'entry': { '@quantity': '2', } }, }, }, { '_id': 'april-ross', 'domain': DOMAIN, 'form': {}, }] export_instance = FormExportInstance( export_format=Format.JSON, domain=DOMAIN, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ StockFormExportColumn( label="StockItem @type", item=StockItem( path=[ PathNode(name='form'), PathNode(name='balance:question-id'), PathNode(name='@type'), ], ), selected=True, ), StockFormExportColumn( label="StockItem @quantity", item=StockItem( path=[ PathNode(name='form'), PathNode(name='balance:question-id'), PathNode(name='entry'), PathNode(name='@quantity'), ], ), selected=True, ), ] )] ) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [u'StockItem @type', u'StockItem @quantity'], u'rows': [ ['question-id', '2'], ['question-id', '2'], [MISSING_VALUE, MISSING_VALUE], [MISSING_VALUE, MISSING_VALUE], ], } } )
def setUpClass(cls): super(TestConvertStockFormExport, cls).setUpClass() cls.schema = FormExportDataSchema( domain=cls.domain, group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ StockItem( path=[ PathNode(name='form'), PathNode(name='transfer:questionid'), PathNode(name='entry'), PathNode(name='@id'), ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ], last_occurrences={cls.app_id: 2}, ), ExportGroupSchema( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True) ], items=[ StockItem( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='transfer:questionid'), PathNode(name='entry'), PathNode(name='@id'), ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ], last_occurrences={cls.app_id: 2}, ), ])
def test_form_stock_columns(self, export_save): """Ensure that we can export stock properties in a form export""" docs = [{ '_id': 'simone-biles', 'domain': DOMAIN, 'form': { 'balance': [{ '@type': 'question-id', 'entry': { '@quantity': '2', } }, { '@type': 'other-question-id', 'entry': { '@quantity': '3', } }] }, }, { '_id': 'sam-mikulak', 'domain': DOMAIN, 'form': { 'balance': { '@type': 'question-id', 'entry': { '@quantity': '2', } }, }, }, { '_id': 'kerri-walsh', 'domain': DOMAIN, 'form': { 'balance': { '@type': 'other-question-id', 'entry': { '@quantity': '2', } }, }, }, { '_id': 'april-ross', 'domain': DOMAIN, 'form': {}, }] export_instance = FormExportInstance( export_format=Format.JSON, domain=DOMAIN, tables=[ TableConfiguration( label="My table", selected=True, path=[], columns=[ StockFormExportColumn( label="StockItem @type", item=StockItem(path=[ PathNode(name='form'), PathNode(name='balance:question-id'), PathNode(name='@type'), ], ), selected=True, ), StockFormExportColumn( label="StockItem @quantity", item=StockItem(path=[ PathNode(name='form'), PathNode(name='balance:question-id'), PathNode(name='entry'), PathNode(name='@quantity'), ], ), selected=True, ), ]) ]) assert_instance_gives_results( docs, export_instance, { 'My table': { 'headers': ['StockItem @type', 'StockItem @quantity'], 'rows': [ ['question-id', '2'], ['question-id', '2'], [MISSING_VALUE, MISSING_VALUE], [MISSING_VALUE, MISSING_VALUE], ], } }) self.assertTrue(export_save.called)
def test_nested_repeat_conversion(self, _, __): instance, _ = self._convert_form_export('repeat_nested') self.assertEqual(instance.name, 'Nested Repeat') # Check for first repeat table table = instance.get_table( [PathNode(name='form'), PathNode(name='repeat', is_repeat=True)]) self.assertTrue(table.selected) self.assertEqual(table.label, 'Repeat: One') index, column = table.get_column([ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='question2') ], 'ExportItem', None) self.assertEqual(column.label, 'Modified Question Two') self.assertEqual(column.selected, True) # Check for second repeat table table = instance.get_table([ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='repeat_nested', is_repeat=True) ], ) self.assertEqual(table.label, 'Repeat: One.#.Two') self.assertTrue(table.selected) index, column = table.get_column( [ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='repeat_nested', is_repeat=True), PathNode(name='nested') ], 'ExportItem', None, ) self.assertEqual(column.label, 'Modified Nested') self.assertEqual(column.selected, True)
from corehq.apps.export.const import ( CASE_NAME_TRANSFORM, OWNER_ID_TRANSFORM, USERNAME_TRANSFORM, ) from corehq.apps.export.models import PathNode # Mapping from old properties to new. Can delete once all exports have been migrated FORM_PROPERTY_MAPPING = { ("form.case.@user_id", None): ([PathNode(name='form'), PathNode(name='meta'), PathNode(name='userID')], None), ("form.case.@xmlns", None): ([PathNode(name="xmlns")], None), ("form.case.create.case_name", None): ([ PathNode(name='form'), PathNode(name='case'), PathNode(name='create'), PathNode(name='case_name') ], None), ("form.case.create.case_type", None): ([ PathNode(name='form'), PathNode(name='case'), PathNode(name='create'), PathNode(name='case_type') ], None), ("form.case.create.owner_id", None): ([ PathNode(name='form'), PathNode(name='case'), PathNode(name='create'), PathNode(name='owner_id')
def test_mismatched_value_type(self): doc = {'external_blobs': {}, 'photo': "this clearly isn't a photo"} result = self.column.get_value('my-domain', "this clearly isn't a photo", doc, [PathNode(name='form')]) self.assertEqual(result, "this clearly isn't a photo")
def test_stock_conversion(self, _): instance, _ = self._convert_case_export('stock') table = instance.get_table(MAIN_TABLE) path = [PathNode(name='stock')] index, column = table.get_column(path, 'ExportItem', None) self.assertTrue(column.selected)
def test_multiple_write_export_instance_calls(self): """ Confirm that calling _write_export_instance() multiple times (as part of a bulk export) works as expected. """ export_instances = [ FormExportInstance( # export_format=Format.JSON, tables=[ TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Q3", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q3')], ), selected=True, ), ] ), ] ), FormExportInstance( # export_format=Format.JSON, tables=[ TableConfiguration( label="My other table", selected=True, path=[PathNode(name="form", is_repeat=False), PathNode(name="q2", is_repeat=False)], columns=[ ExportColumn( label="Q4", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4')], ), selected=True, ), ] ) ] ) ] writer = _Writer(get_writer(Format.JSON)) with writer.open(export_instances): _write_export_instance(writer, export_instances[0], self.docs) _write_export_instance(writer, export_instances[1], self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'Export1-My table': { u'headers': [u'Q3'], u'rows': [[u'baz'], [u'bop']], }, u'Export2-My other table': { u'headers': [u'Q4'], u'rows': [[u'bar'], [u'boop']], } } )
def setUpClass(cls): super(TestConvertSavedExportSchemaToFormExportInstance, cls).setUpClass() cls.schema = FormExportDataSchema( domain=cls.domain, group_schemas=[ ExportGroupSchema( path=MAIN_TABLE, items=[ ExportItem( path=[ PathNode(name='form'), PathNode(name='question1') ], label='Question 1 Not updated', last_occurrences={cls.app_id: 3}, ), ExportItem( path=[ PathNode(name='form'), PathNode(name='deid_id') ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ExportItem( path=[ PathNode(name='form'), PathNode(name='deid_date') ], label='Question 1', last_occurrences={cls.app_id: 3}, ), ], last_occurrences={cls.app_id: 3}, ), ExportGroupSchema( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True) ], items=[ ExportItem( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='question2') ], label='Question 2', last_occurrences={cls.app_id: 2}, ) ], last_occurrences={cls.app_id: 2}, ), ExportGroupSchema( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='repeat_nested', is_repeat=True), ], items=[ ExportItem( path=[ PathNode(name='form'), PathNode(name='repeat', is_repeat=True), PathNode(name='repeat_nested', is_repeat=True), PathNode(name='nested'), ], label='Nested Repeat', last_occurrences={cls.app_id: 2}, ) ], last_occurrences={cls.app_id: 2}, ), ], )
def test_missing_value(self): doc = {'external_blobs': {}, 'photo': None} result = self.column.get_value('my-domain', '1234', doc, [PathNode(name='form')]) self.assertEqual(result, MISSING_VALUE)