def test_get_export_file(self): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Foo column", item=ExportItem( path=[PathNode(name="foo")] ), selected=True, ), ExportColumn( label="Bar column", item=ExportItem( path=[PathNode(name="bar")] ), selected=True, ) ] )] ), ], [] # No filters ) with export_file as export: self.assertEqual( json.loads(export), { u'My table': { u'headers': [ u'Foo column', u'Bar column'], u'rows': [ [u'apple', u'banana'], [u'apple', u'banana'], [u'apple', u'banana'], ], } } )
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60): export_file = get_export_file(export_instances, filters) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name escaped_filename = escape_quotes('%s.%s' % (filename, file_format.extension)) payload = export_file.file.payload expose_cached_download( payload, expiry, ".{}".format(file_format.extension), mimetype=file_format.mimetype, content_disposition='attachment; filename="%s"' % escaped_filename, download_id=download_id, ) export_file.file.delete()
def test_export_transforms(self, _): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="DEID Date Transform column", item=ExportItem( path=[PathNode(name="date")] ), selected=True, deid_transform=DEID_DATE_TRANSFORM, ) ] )] ), ], [] # No filters ) with export_file as export: export_dict = json.loads(export) export_dict['My table']['rows'].sort() self.assertEqual( export_dict, { u'My table': { u'headers': [ u'DEID Date Transform column [sensitive]', ], u'rows': [ [None], [u'2016-04-07'], [u'2016-04-27'], # offset by 3 since that's the mocked random offset ], } } )
def test_selected_false(self): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=False, path=[], columns=[] )] ), ], [] # No filters ) with export_file as export: self.assertEqual(json.loads(export), {})
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60): export_file = get_export_file( export_instances, filters, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None ) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name escaped_filename = escape_quotes('%s.%s' % (filename, file_format.extension)) escaped_filename = urllib.quote(escaped_filename.encode('utf8')) payload = export_file.file.payload expose_cached_download( payload, expiry, ".{}".format(file_format.extension), mimetype=file_format.mimetype, content_disposition='attachment; filename="%s"' % escaped_filename, download_id=download_id, ) export_file.file.delete()
def test_simple_bulk_export(self): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", path=MAIN_TABLE, columns=[ ExportColumn( label="Foo column", item=ExportItem( path=["foo"] ), selected=True, ), ] )] ), CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", path=MAIN_TABLE, columns=[ ExportColumn( label="Bar column", item=ExportItem( path=["bar"] ), selected=True, ) ] )] ), ], [] # No filters ) expected = { 'My table': { "A1": "Foo column", "A2": "apple", "A3": "apple", "A4": "apple", }, "My table1": { "A1": "Bar column", "A2": "banana", "A3": "banana", "A4": "banana", }, } with export_file as export: wb = load_workbook(StringIO(export)) self.assertEqual(wb.get_sheet_names(), ["My table", "My table1"]) for sheet in expected.keys(): for cell in expected[sheet].keys(): self.assertEqual( wb[sheet][cell].value, expected[sheet][cell], 'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format( sheet, cell, expected[sheet][cell], wb[sheet][cell].value ) )
def test_simple_bulk_export(self, export_save): with TransientTempfile() as temp_path: export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( selected=True, label="My table", path=MAIN_TABLE, columns=[ ExportColumn( label="Foo column", item=ExportItem( path=[PathNode(name="foo")] ), selected=True, ), ] )] ), CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=MAIN_TABLE, columns=[ ExportColumn( label="Bar column", item=ExportItem( path=[PathNode(name="bar")] ), selected=True, ) ] )] ), ], [], # No filters temp_path, ) expected = { 'Export1-My table': { "A1": "Foo column", "A2": "apple", "A3": "apple", "A4": "apple", }, "Export2-My table": { "A1": "Bar column", "A2": "banana", "A3": "banana", "A4": "banana", }, } with export_file as export: wb = load_workbook(export) self.assertEqual(wb.get_sheet_names(), ["Export1-My table", "Export2-My table"]) for sheet in expected.keys(): for cell in expected[sheet].keys(): self.assertEqual( wb[sheet][cell].value, expected[sheet][cell], 'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format( sheet, cell, expected[sheet][cell], wb[sheet][cell].value ) ) self.assertTrue(export_save.called)
def test_simple_bulk_export(self, export_save): with TransientTempfile() as temp_path: export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[ TableConfiguration( selected=True, label="My table", path=MAIN_TABLE, columns=[ ExportColumn( label="Foo column", item=ExportItem( path=[PathNode(name="foo")]), selected=True, ), ]) ]), CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[ TableConfiguration( label="My table", selected=True, path=MAIN_TABLE, columns=[ ExportColumn( label="Bar column", item=ExportItem( path=[PathNode(name="bar")]), selected=True, ) ]) ]), ], [], # No filters temp_path, ) expected = { 'Export1-My table': { "A1": "Foo column", "A2": "apple", "A3": "apple", "A4": "apple", }, "Export2-My table": { "A1": "Bar column", "A2": "banana", "A3": "banana", "A4": "banana", }, } with export_file as export: wb = load_workbook(export) self.assertEqual(wb.get_sheet_names(), ["Export1-My table", "Export2-My table"]) for sheet in expected.keys(): for cell in expected[sheet].keys(): self.assertEqual( wb[sheet][cell].value, expected[sheet][cell], 'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"' .format(sheet, cell, expected[sheet][cell], wb[sheet][cell].value)) self.assertTrue(export_save.called)
def test(self): schema = FormExportDataSchema.generate_schema_from_builds( self.domain, self.app._id, self.form_xmlns, only_process_current_builds=True, ) for group_schema in schema.group_schemas: # group_schema is an instance of ExportGroupSchem # group_schema.items is an array of ExportItems subclasses path = [node.name for node in group_schema.path] if path == []: main_group_schema = group_schema elif path == ['form', 'babies']: baby_repeat_group_schema = group_schema assertContainsExportItems( [ # Verify that a simple form question appears in the schema ('form.how_are_you_today', 'How are you today?'), ('form.how_many_babies', 'How many babies?'), ('form.add_a_prescription', 'Add a prescription?'), ('form.voucher-name', '#form/voucher-name'), ('form.is_this_a_delivery', 'Is this a delivery?'), # Verify that the main parent case updates appear (case type "mom") ('form.case.@case_id', 'case.@case_id'), ('form.case.@date_modified', 'case.@date_modified'), ('form.case.@user_id', 'case.@user_id'), ('form.case.update.last_status', 'case.update.last_status'), # Verify that we see case updates for save-to-case cases # These are already in the form schema, so they get interpreted like any other question ('form.prescription.prescription.case.close', '#form/prescription/prescription/case/close'), ('form.prescription.prescription.case.create.case_name', '#form/prescription/prescription/case/create/case_name'), ('form.prescription.prescription.case.create.case_type', '#form/prescription/prescription/case/create/case_type'), ('form.prescription.prescription.case.update.number_of_babies', '#form/prescription/prescription/case/update/number_of_babies' ), ('form.prescription.prescription_name', '#form/prescription/prescription_name'), ('form.prescription.prescription.case.index.parent', '#form/prescription/prescription/case/index/parent'), # # Verify that we see updates from subcases not in repeat groups (case type "voucher") ('form.subcase_0.case.@case_id', 'subcase_0.@case_id'), ('form.subcase_0.case.@date_modified', 'subcase_0.@date_modified'), ('form.subcase_0.case.@user_id', 'subcase_0.@user_id'), ('form.subcase_0.case.create.case_name', 'subcase_0.create.case_name'), ('form.subcase_0.case.create.case_type', 'subcase_0.create.case_type'), ('form.subcase_0.case.create.owner_id', 'subcase_0.create.owner_id'), ('form.subcase_0.case.index.parent.#text', 'subcase_0.index.#text'), ('form.subcase_0.case.index.parent.@case_type', 'subcase_0.index.@case_type'), ('form.subcase_0.case.update.how_many_babies', 'subcase_0.update.how_many_babies'), ], main_group_schema) # Verify that we see updates from subcases in repeat groups (case type "baby") assertContainsExportItems([ ('form.babies.case.@case_id', 'subcase_1.@case_id'), ('form.babies.case.@date_modified', 'subcase_1.@date_modified'), ('form.babies.case.@user_id', 'subcase_1.@user_id'), ('form.babies.case.create.case_name', 'subcase_1.create.case_name'), ('form.babies.case.create.case_type', 'subcase_1.create.case_type'), ('form.babies.case.create.owner_id', 'subcase_1.create.owner_id'), ('form.babies.case.update.eye_color', 'subcase_1.update.eye_color'), ('form.babies.case.index.parent.#text', 'subcase_1.index.#text'), ('form.babies.case.index.parent.@case_type', 'subcase_1.index.@case_type'), ('form.babies.eye_color', 'Eye color?'), ('form.babies.whats_the_babys_name', "What's the baby's name?"), ], baby_repeat_group_schema) instance = FormExportInstance.generate_instance_from_schema(schema) instance.export_format = Format.JSON # make everything show up in the export for table in instance.tables: table.selected = True for column in table.columns: column.selected = True with patch('corehq.apps.export.export.get_export_documents') as docs: docs.return_value = self.form_es_response export_file = get_export_file([instance], []) with export_file as export: export_data = json.loads(export.read()) def get_form_data(table): headers = export_data[table]['headers'] return [ dict(zip(headers, row)) for row in export_data[table]['rows'] ] self.assertDictContainsSubset( { # normal form questions "form.add_a_prescription": "yes_then_close", "form.how_are_you_today": "fine_thanks", "form.how_many_babies": "2", "form.is_this_a_delivery": "yes", "form.voucher-name": "Petunia2017-08-29", # standard case update "form.case.update.last_status": "fine_thanks", "form.case.@case_id": "71626d9c-2d05-491f-81d9-becf8566618a", "form.case.@user_id": "853a24735ba89a3019ced7e3153dc60d", # save-to-case properties "form.prescription.prescription.case.close": "True", "form.prescription.prescription.case.create.case_name": "Petunia-prescription-2017-08-29", "form.prescription.prescription.case.create.case_type": "prescription", "form.prescription.prescription.case.update.number_of_babies": "2", "form.prescription.prescription_name": "Petunia-prescription-2017-08-29", "form.prescription.prescription.case.index.parent": "71626d9c-2d05-491f-81d9-becf8566618a", # non-repeating subcase actions "form.subcase_0.case.@case_id": "16954d55-a9be-40dd-98a8-dc7fae9c7ed6", "form.subcase_0.case.@date_modified": "2017-08-29 11:19:40", "form.subcase_0.case.@user_id": "853a24735ba89a3019ced7e3153dc60d", "form.subcase_0.case.create.case_name": "Petunia2017-08-29", "form.subcase_0.case.create.case_type": "voucher", "form.subcase_0.case.create.owner_id": "853a24735ba89a3019ced7e3153dc60d", "form.subcase_0.case.update.how_many_babies": "2", "form.subcase_0.case.index.parent.#text": "71626d9c-2d05-491f-81d9-becf8566618a", "form.subcase_0.case.index.parent.@case_type": "mom", }, get_form_data('Forms')[0]) self.assertDictContainsSubset( { "form.babies.eye_color": "brown", "form.babies.whats_the_babys_name": "Bob", "form.babies.case.@case_id": "5539bd9d-d5d6-44c8-8f78-6915f16b6907", "form.babies.case.@date_modified": "2017-08-29 11:19:40", "form.babies.case.@user_id": "853a24735ba89a3019ced7e3153dc60d", "form.babies.case.create.case_name": "Bob", "form.babies.case.create.case_type": "baby", "form.babies.case.create.owner_id": "853a24735ba89a3019ced7e3153dc60d", "form.babies.case.index.parent.#text": "71626d9c-2d05-491f-81d9-becf8566618a", "form.babies.case.index.parent.@case_type": "mom", "form.babies.case.update.eye_color": "brown", }, get_form_data('Repeat- babies')[0])
def get_export_json(export_instance): with TransientTempfile() as temp_path: export_file = get_export_file([export_instance], [], temp_path) with export_file as export: return json.loads(export.read())
def get_export_json(export_instance): with TransientTempfile() as temp_path: export_file = get_export_file([export_instance], [], temp_path) with export_file as export: return json.loads(export.read())