def test_get_export_file(self):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
],
[] # No filters
)
with export_file as export:
self.assertEqual(
json.loads(export),
{
u'My table': {
u'headers': [
u'Foo column',
u'Bar column'],
u'rows': [
[u'apple', u'banana'],
[u'apple', u'banana'],
[u'apple', u'banana'],
],
}
}
)
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60):
export_file = get_export_file(export_instances, filters)
file_format = Format.from_format(export_file.format)
filename = filename or export_instances[0].name
escaped_filename = escape_quotes('%s.%s' % (filename, file_format.extension))
payload = export_file.file.payload
expose_cached_download(
payload,
expiry,
".{}".format(file_format.extension),
mimetype=file_format.mimetype,
content_disposition='attachment; filename="%s"' % escaped_filename,
download_id=download_id,
)
export_file.file.delete()
def test_export_transforms(self, _):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="DEID Date Transform column",
item=ExportItem(
path=[PathNode(name="date")]
),
selected=True,
deid_transform=DEID_DATE_TRANSFORM,
)
]
)]
),
],
[] # No filters
)
with export_file as export:
export_dict = json.loads(export)
export_dict['My table']['rows'].sort()
self.assertEqual(
export_dict,
{
u'My table': {
u'headers': [
u'DEID Date Transform column [sensitive]',
],
u'rows': [
[None],
[u'2016-04-07'],
[u'2016-04-27'], # offset by 3 since that's the mocked random offset
],
}
}
)
def test_selected_false(self):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=False,
path=[],
columns=[]
)]
),
],
[] # No filters
)
with export_file as export:
self.assertEqual(json.loads(export), {})
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60):
export_file = get_export_file(
export_instances,
filters,
# We don't have a great way to calculate progress if it's a bulk download,
# so only track the progress for single instance exports.
progress_tracker=populate_export_download_task if len(export_instances) == 1 else None
)
file_format = Format.from_format(export_file.format)
filename = filename or export_instances[0].name
escaped_filename = escape_quotes('%s.%s' % (filename, file_format.extension))
escaped_filename = urllib.quote(escaped_filename.encode('utf8'))
payload = export_file.file.payload
expose_cached_download(
payload,
expiry,
".{}".format(file_format.extension),
mimetype=file_format.mimetype,
content_disposition='attachment; filename="%s"' % escaped_filename,
download_id=download_id,
)
export_file.file.delete()
def test_simple_bulk_export(self):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=["foo"]
),
selected=True,
),
]
)]
),
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Bar column",
item=ExportItem(
path=["bar"]
),
selected=True,
)
]
)]
),
],
[] # No filters
)
expected = {
'My table': {
"A1": "Foo column",
"A2": "apple",
"A3": "apple",
"A4": "apple",
},
"My table1": {
"A1": "Bar column",
"A2": "banana",
"A3": "banana",
"A4": "banana",
},
}
with export_file as export:
wb = load_workbook(StringIO(export))
self.assertEqual(wb.get_sheet_names(), ["My table", "My table1"])
for sheet in expected.keys():
for cell in expected[sheet].keys():
self.assertEqual(
wb[sheet][cell].value,
expected[sheet][cell],
'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format(
sheet, cell, expected[sheet][cell], wb[sheet][cell].value
)
)
def test_simple_bulk_export(self, export_save):
with TransientTempfile() as temp_path:
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
selected=True,
label="My table",
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
]
)]
),
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
],
[], # No filters
temp_path,
)
expected = {
'Export1-My table': {
"A1": "Foo column",
"A2": "apple",
"A3": "apple",
"A4": "apple",
},
"Export2-My table": {
"A1": "Bar column",
"A2": "banana",
"A3": "banana",
"A4": "banana",
},
}
with export_file as export:
wb = load_workbook(export)
self.assertEqual(wb.get_sheet_names(), ["Export1-My table", "Export2-My table"])
for sheet in expected.keys():
for cell in expected[sheet].keys():
self.assertEqual(
wb[sheet][cell].value,
expected[sheet][cell],
'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format(
sheet, cell, expected[sheet][cell], wb[sheet][cell].value
)
)
self.assertTrue(export_save.called)
def test_simple_bulk_export(self, export_save):
with TransientTempfile() as temp_path:
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
selected=True,
label="My table",
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]),
selected=True,
),
])
]),
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]),
selected=True,
)
])
]),
],
[], # No filters
temp_path,
)
expected = {
'Export1-My table': {
"A1": "Foo column",
"A2": "apple",
"A3": "apple",
"A4": "apple",
},
"Export2-My table": {
"A1": "Bar column",
"A2": "banana",
"A3": "banana",
"A4": "banana",
},
}
with export_file as export:
wb = load_workbook(export)
self.assertEqual(wb.get_sheet_names(),
["Export1-My table", "Export2-My table"])
for sheet in expected.keys():
for cell in expected[sheet].keys():
self.assertEqual(
wb[sheet][cell].value, expected[sheet][cell],
'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'
.format(sheet, cell, expected[sheet][cell],
wb[sheet][cell].value))
self.assertTrue(export_save.called)
def test(self):
schema = FormExportDataSchema.generate_schema_from_builds(
self.domain,
self.app._id,
self.form_xmlns,
only_process_current_builds=True,
)
for group_schema in schema.group_schemas:
# group_schema is an instance of ExportGroupSchem
# group_schema.items is an array of ExportItems subclasses
path = [node.name for node in group_schema.path]
if path == []:
main_group_schema = group_schema
elif path == ['form', 'babies']:
baby_repeat_group_schema = group_schema
assertContainsExportItems(
[
# Verify that a simple form question appears in the schema
('form.how_are_you_today', 'How are you today?'),
('form.how_many_babies', 'How many babies?'),
('form.add_a_prescription', 'Add a prescription?'),
('form.voucher-name', '#form/voucher-name'),
('form.is_this_a_delivery', 'Is this a delivery?'),
# Verify that the main parent case updates appear (case type "mom")
('form.case.@case_id', 'case.@case_id'),
('form.case.@date_modified', 'case.@date_modified'),
('form.case.@user_id', 'case.@user_id'),
('form.case.update.last_status', 'case.update.last_status'),
# Verify that we see case updates for save-to-case cases
# These are already in the form schema, so they get interpreted like any other question
('form.prescription.prescription.case.close',
'#form/prescription/prescription/case/close'),
('form.prescription.prescription.case.create.case_name',
'#form/prescription/prescription/case/create/case_name'),
('form.prescription.prescription.case.create.case_type',
'#form/prescription/prescription/case/create/case_type'),
('form.prescription.prescription.case.update.number_of_babies',
'#form/prescription/prescription/case/update/number_of_babies'
),
('form.prescription.prescription_name',
'#form/prescription/prescription_name'),
('form.prescription.prescription.case.index.parent',
'#form/prescription/prescription/case/index/parent'),
# # Verify that we see updates from subcases not in repeat groups (case type "voucher")
('form.subcase_0.case.@case_id', 'subcase_0.@case_id'),
('form.subcase_0.case.@date_modified',
'subcase_0.@date_modified'),
('form.subcase_0.case.@user_id', 'subcase_0.@user_id'),
('form.subcase_0.case.create.case_name',
'subcase_0.create.case_name'),
('form.subcase_0.case.create.case_type',
'subcase_0.create.case_type'),
('form.subcase_0.case.create.owner_id',
'subcase_0.create.owner_id'),
('form.subcase_0.case.index.parent.#text',
'subcase_0.index.#text'),
('form.subcase_0.case.index.parent.@case_type',
'subcase_0.index.@case_type'),
('form.subcase_0.case.update.how_many_babies',
'subcase_0.update.how_many_babies'),
],
main_group_schema)
# Verify that we see updates from subcases in repeat groups (case type "baby")
assertContainsExportItems([
('form.babies.case.@case_id', 'subcase_1.@case_id'),
('form.babies.case.@date_modified', 'subcase_1.@date_modified'),
('form.babies.case.@user_id', 'subcase_1.@user_id'),
('form.babies.case.create.case_name',
'subcase_1.create.case_name'),
('form.babies.case.create.case_type',
'subcase_1.create.case_type'),
('form.babies.case.create.owner_id', 'subcase_1.create.owner_id'),
('form.babies.case.update.eye_color',
'subcase_1.update.eye_color'),
('form.babies.case.index.parent.#text', 'subcase_1.index.#text'),
('form.babies.case.index.parent.@case_type',
'subcase_1.index.@case_type'),
('form.babies.eye_color', 'Eye color?'),
('form.babies.whats_the_babys_name', "What's the baby's name?"),
], baby_repeat_group_schema)
instance = FormExportInstance.generate_instance_from_schema(schema)
instance.export_format = Format.JSON
# make everything show up in the export
for table in instance.tables:
table.selected = True
for column in table.columns:
column.selected = True
with patch('corehq.apps.export.export.get_export_documents') as docs:
docs.return_value = self.form_es_response
export_file = get_export_file([instance], [])
with export_file as export:
export_data = json.loads(export.read())
def get_form_data(table):
headers = export_data[table]['headers']
return [
dict(zip(headers, row)) for row in export_data[table]['rows']
]
self.assertDictContainsSubset(
{
# normal form questions
"form.add_a_prescription": "yes_then_close",
"form.how_are_you_today": "fine_thanks",
"form.how_many_babies": "2",
"form.is_this_a_delivery": "yes",
"form.voucher-name": "Petunia2017-08-29",
# standard case update
"form.case.update.last_status": "fine_thanks",
"form.case.@case_id": "71626d9c-2d05-491f-81d9-becf8566618a",
"form.case.@user_id": "853a24735ba89a3019ced7e3153dc60d",
# save-to-case properties
"form.prescription.prescription.case.close": "True",
"form.prescription.prescription.case.create.case_name":
"Petunia-prescription-2017-08-29",
"form.prescription.prescription.case.create.case_type":
"prescription",
"form.prescription.prescription.case.update.number_of_babies":
"2",
"form.prescription.prescription_name":
"Petunia-prescription-2017-08-29",
"form.prescription.prescription.case.index.parent":
"71626d9c-2d05-491f-81d9-becf8566618a",
# non-repeating subcase actions
"form.subcase_0.case.@case_id":
"16954d55-a9be-40dd-98a8-dc7fae9c7ed6",
"form.subcase_0.case.@date_modified": "2017-08-29 11:19:40",
"form.subcase_0.case.@user_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.subcase_0.case.create.case_name": "Petunia2017-08-29",
"form.subcase_0.case.create.case_type": "voucher",
"form.subcase_0.case.create.owner_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.subcase_0.case.update.how_many_babies": "2",
"form.subcase_0.case.index.parent.#text":
"71626d9c-2d05-491f-81d9-becf8566618a",
"form.subcase_0.case.index.parent.@case_type": "mom",
},
get_form_data('Forms')[0])
self.assertDictContainsSubset(
{
"form.babies.eye_color": "brown",
"form.babies.whats_the_babys_name": "Bob",
"form.babies.case.@case_id":
"5539bd9d-d5d6-44c8-8f78-6915f16b6907",
"form.babies.case.@date_modified": "2017-08-29 11:19:40",
"form.babies.case.@user_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.babies.case.create.case_name": "Bob",
"form.babies.case.create.case_type": "baby",
"form.babies.case.create.owner_id":
"853a24735ba89a3019ced7e3153dc60d",
"form.babies.case.index.parent.#text":
"71626d9c-2d05-491f-81d9-becf8566618a",
"form.babies.case.index.parent.@case_type": "mom",
"form.babies.case.update.eye_color": "brown",
},
get_form_data('Repeat- babies')[0])
def get_export_json(export_instance):
with TransientTempfile() as temp_path:
export_file = get_export_file([export_instance], [], temp_path)
with export_file as export:
return json.loads(export.read())
def get_export_json(export_instance):
with TransientTempfile() as temp_path:
export_file = get_export_file([export_instance], [], temp_path)
with export_file as export:
return json.loads(export.read())
