def test_revert_new_exports_restore_old(self):
saved_export_schema = SavedExportSchema(index=['my-domain', 'xmlns'])
saved_export_schema.doc_type += DELETED_SUFFIX
saved_export_schema.save()
self.new_exports[
0].legacy_saved_export_schema_id = saved_export_schema._id
reverted = revert_new_exports(self.new_exports)
self.assertEqual(len(reverted), 1)
self.assertFalse(reverted[0].doc_type.endswith(DELETED_SUFFIX))
saved_export_schema.delete()
class TestConvertIndexToPath(SimpleTestCase):
"""Test the conversion of old style index to new style path"""
@generate_cases(
[
('form.question1', [PathNode(name='form'),
PathNode(name='question1')]),
('#', MAIN_TABLE),
('#.form.question1.#',
[PathNode(name='form'),
PathNode(name='question1', is_repeat=True)]), # Repeat group
],
TestConvertIndexToPath)
def test_convert_index_to_path_nodes(self, index, path):
self.assertEqual(_convert_index_to_path_nodes(index), path)
from corehq.apps.export.models import (
ExportColumn,
ExportItem,
PathNode,
StockExportColumn,
RowNumberColumn,
SplitGPSExportColumn,
GeopointItem,
)
# System properties to be displayed above the form questions
TOP_MAIN_FORM_TABLE_PROPERTIES = [
RowNumberColumn(
tags=[PROPERTY_TAG_ROW],
label="number",
item=ExportItem(path=[PathNode(name='number')]),
selected=True,
),
ExportColumn(
tags=[PROPERTY_TAG_INFO],
label="formid",
item=ExportItem(path=[
PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='instanceID')
]),
help_text=_('Unique identifier of the form submission'),
selected=True,
)
]
def setUpClass(cls):
super(TestExportInstanceFromSavedInstance, cls).setUpClass()
cls.app_id = '1234'
cls.schema = FormExportDataSchema(group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question1')
],
label='Question 1',
last_occurrences={
cls.app_id: 3,
},
),
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='@case_id')
],
label='@case_id',
tag=PROPERTY_TAG_CASE,
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
], )
cls.new_schema = FormExportDataSchema(group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question1')
],
label='Question 1',
last_occurrences={
cls.app_id: 3,
},
),
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='@case_id')
],
label='@case_id',
tag=PROPERTY_TAG_CASE,
last_occurrences={
cls.app_id: 3,
},
),
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question3')
],
label='Question 3',
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
ExportGroupSchema(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='q2')
],
label='Question 2',
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
], )
def test_get_column(self):
table_configuration = TableConfiguration(
path=[PathNode(name='form', is_repeat=False), PathNode(name="repeat1", is_repeat=True)],
columns=[
ExportColumn(
item=ScalarItem(
path=[
PathNode(name='form'),
PathNode(name='repeat1', is_repeat=True),
PathNode(name='q1')
],
)
),
ExportColumn(
item=ScalarItem(
path=[
PathNode(name="form"),
PathNode(name="user_id"),
],
transform=USERNAME_TRANSFORM
)
),
ExportColumn(
item=ScalarItem(
path=[
PathNode(name='form'),
PathNode(name='repeat1', is_repeat=True),
PathNode(name='q2')
],
)
),
]
)
index, column = table_configuration.get_column(
[
PathNode(name='form'),
PathNode(name='repeat1', is_repeat=True),
PathNode(name='q1')
],
'ScalarItem',
None,
)
self.assertEqual(
column.item.path,
[
PathNode(name='form'),
PathNode(name='repeat1', is_repeat=True),
PathNode(name='q1')
]
)
self.assertEqual(index, 0)
index, column = table_configuration.get_column(
[
PathNode(name='form'),
PathNode(name='repeat1', is_repeat=True),
PathNode(name='DoesNotExist')
],
'ScalarItem',
None,
)
self.assertIsNone(column)
# Verify that get_column ignores deid transforms
index, column = table_configuration.get_column(
[PathNode(name="form"), PathNode(name="user_id")],
'ScalarItem',
USERNAME_TRANSFORM
)
self.assertIsNotNone(column)
self.assertEqual(index, 1)
PathNode,
StockExportColumn,
RowNumberColumn,
SplitGPSExportColumn,
GeopointItem,
ScalarItem,
)
# System properties to be displayed above the form questions
from corehq.apps.userreports.datatypes import DATA_TYPE_DATETIME, DATA_TYPE_STRING
TOP_MAIN_FORM_TABLE_PROPERTIES = [
RowNumberColumn(
tags=[PROPERTY_TAG_ROW],
label="number",
item=ExportItem(path=[PathNode(name='number')]),
selected=True,
),
ExportColumn(
tags=[PROPERTY_TAG_INFO],
label="formid",
item=ExportItem(path=[
PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='instanceID')
]),
help_text=_('Unique identifier of the form submission'),
selected=True,
)
]
def test_empty_string(self):
doc = {'external_blobs': {}, 'photo': ''}
result = self.column.get_value('my-domain', '1234', doc,
[PathNode(name='form')])
self.assertEqual(result, '')
def test_simple_bulk_export(self):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
selected=True,
label="My table",
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
]
)]
),
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
],
[] # No filters
)
expected = {
'Export1-My table': {
"A1": "Foo column",
"A2": "apple",
"A3": "apple",
"A4": "apple",
},
"Export2-My table": {
"A1": "Bar column",
"A2": "banana",
"A3": "banana",
"A4": "banana",
},
}
with export_file as export:
wb = load_workbook(export)
self.assertEqual(wb.get_sheet_names(), ["Export1-My table", "Export2-My table"])
for sheet in expected.keys():
for cell in expected[sheet].keys():
self.assertEqual(
wb[sheet][cell].value,
expected[sheet][cell],
'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format(
sheet, cell, expected[sheet][cell], wb[sheet][cell].value
)
)
def test_multiple_write_export_instance_calls(self, export_save):
"""
Confirm that calling _write_export_instance() multiple times
(as part of a bulk export) works as expected.
"""
export_instances = [
FormExportInstance(tables=[
TableConfiguration(label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Q3",
item=ScalarItem(path=[
PathNode(name='form'),
PathNode(name='q3')
], ),
selected=True,
),
]),
]),
FormExportInstance(tables=[
TableConfiguration(label="My other table",
selected=True,
path=[
PathNode(name="form", is_repeat=False),
PathNode(name="q2", is_repeat=False)
],
columns=[
ExportColumn(
label="Q4",
item=ScalarItem(path=[
PathNode(name='form'),
PathNode(name='q2'),
PathNode(name='q4')
], ),
selected=True,
),
])
]),
FormExportInstance(tables=[
TableConfiguration(label="My other table",
selected=True,
path=[
PathNode(name="form", is_repeat=False),
PathNode(name="q2", is_repeat=False)
],
columns=[
ExportColumn(
label="Q4",
item=ScalarItem(path=[
PathNode(name='form'),
PathNode(name='q2'),
PathNode(name='q4')
], ),
selected=True,
),
])
])
]
with TransientTempfile() as temp_path:
writer = _ExportWriter(get_writer(Format.JSON), temp_path)
with writer.open(export_instances):
write_export_instance(writer, export_instances[0], self.docs)
write_export_instance(writer, export_instances[1], self.docs)
write_export_instance(writer, export_instances[2], self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()), {
'My table': {
'headers': ['Q3'],
'rows': [['baz'], ['bop']],
},
'Export2-My other table': {
'headers': ['Q4'],
'rows': [['bar'], ['boop']],
},
'Export3-My other table': {
'headers': ['Q4'],
'rows': [['bar'], ['boop']],
},
})
self.assertTrue(export_save.called)
def test_form_stock_columns(self):
"""Ensure that we can export stock properties in a form export"""
docs = [{
'_id': 'simone-biles',
'domain': DOMAIN,
'form': {
'balance': [
{
'@type': 'question-id',
'entry': {
'@quantity': '2',
}
}, {
'@type': 'other-question-id',
'entry': {
'@quantity': '3',
}
}]
},
}, {
'_id': 'sam-mikulak',
'domain': DOMAIN,
'form': {
'balance': {
'@type': 'question-id',
'entry': {
'@quantity': '2',
}
},
},
}, {
'_id': 'kerri-walsh',
'domain': DOMAIN,
'form': {
'balance': {
'@type': 'other-question-id',
'entry': {
'@quantity': '2',
}
},
},
}, {
'_id': 'april-ross',
'domain': DOMAIN,
'form': {},
}]
export_instance = FormExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
StockFormExportColumn(
label="StockItem @type",
item=StockItem(
path=[
PathNode(name='form'),
PathNode(name='balance:question-id'),
PathNode(name='@type'),
],
),
selected=True,
),
StockFormExportColumn(
label="StockItem @quantity",
item=StockItem(
path=[
PathNode(name='form'),
PathNode(name='balance:question-id'),
PathNode(name='entry'),
PathNode(name='@quantity'),
],
),
selected=True,
),
]
)]
)
writer = _get_writer([export_instance])
with writer.open([export_instance]):
_write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
u'My table': {
u'headers': [u'StockItem @type', u'StockItem @quantity'],
u'rows': [
['question-id', '2'],
['question-id', '2'],
[MISSING_VALUE, MISSING_VALUE],
[MISSING_VALUE, MISSING_VALUE],
],
}
}
)
def setUpClass(cls):
super(TestConvertStockFormExport, cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
StockItem(
path=[
PathNode(name='form'),
PathNode(name='transfer:questionid'),
PathNode(name='entry'),
PathNode(name='@id'),
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 2},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True)
],
items=[
StockItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='transfer:questionid'),
PathNode(name='entry'),
PathNode(name='@id'),
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 2},
),
])
def test_form_stock_columns(self, export_save):
"""Ensure that we can export stock properties in a form export"""
docs = [{
'_id': 'simone-biles',
'domain': DOMAIN,
'form': {
'balance': [{
'@type': 'question-id',
'entry': {
'@quantity': '2',
}
}, {
'@type': 'other-question-id',
'entry': {
'@quantity': '3',
}
}]
},
}, {
'_id': 'sam-mikulak',
'domain': DOMAIN,
'form': {
'balance': {
'@type': 'question-id',
'entry': {
'@quantity': '2',
}
},
},
}, {
'_id': 'kerri-walsh',
'domain': DOMAIN,
'form': {
'balance': {
'@type': 'other-question-id',
'entry': {
'@quantity': '2',
}
},
},
}, {
'_id': 'april-ross',
'domain': DOMAIN,
'form': {},
}]
export_instance = FormExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
StockFormExportColumn(
label="StockItem @type",
item=StockItem(path=[
PathNode(name='form'),
PathNode(name='balance:question-id'),
PathNode(name='@type'),
], ),
selected=True,
),
StockFormExportColumn(
label="StockItem @quantity",
item=StockItem(path=[
PathNode(name='form'),
PathNode(name='balance:question-id'),
PathNode(name='entry'),
PathNode(name='@quantity'),
], ),
selected=True,
),
])
])
assert_instance_gives_results(
docs, export_instance, {
'My table': {
'headers': ['StockItem @type', 'StockItem @quantity'],
'rows': [
['question-id', '2'],
['question-id', '2'],
[MISSING_VALUE, MISSING_VALUE],
[MISSING_VALUE, MISSING_VALUE],
],
}
})
self.assertTrue(export_save.called)
def test_nested_repeat_conversion(self, _, __):
instance, _ = self._convert_form_export('repeat_nested')
self.assertEqual(instance.name, 'Nested Repeat')
# Check for first repeat table
table = instance.get_table(
[PathNode(name='form'),
PathNode(name='repeat', is_repeat=True)])
self.assertTrue(table.selected)
self.assertEqual(table.label, 'Repeat: One')
index, column = table.get_column([
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='question2')
], 'ExportItem', None)
self.assertEqual(column.label, 'Modified Question Two')
self.assertEqual(column.selected, True)
# Check for second repeat table
table = instance.get_table([
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='repeat_nested', is_repeat=True)
], )
self.assertEqual(table.label, 'Repeat: One.#.Two')
self.assertTrue(table.selected)
index, column = table.get_column(
[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='repeat_nested', is_repeat=True),
PathNode(name='nested')
],
'ExportItem',
None,
)
self.assertEqual(column.label, 'Modified Nested')
self.assertEqual(column.selected, True)
from corehq.apps.export.const import (
CASE_NAME_TRANSFORM,
OWNER_ID_TRANSFORM,
USERNAME_TRANSFORM,
)
from corehq.apps.export.models import PathNode
# Mapping from old properties to new. Can delete once all exports have been migrated
FORM_PROPERTY_MAPPING = {
("form.case.@user_id", None):
([PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='userID')], None),
("form.case.@xmlns", None): ([PathNode(name="xmlns")], None),
("form.case.create.case_name", None): ([
PathNode(name='form'),
PathNode(name='case'),
PathNode(name='create'),
PathNode(name='case_name')
], None),
("form.case.create.case_type", None): ([
PathNode(name='form'),
PathNode(name='case'),
PathNode(name='create'),
PathNode(name='case_type')
], None),
("form.case.create.owner_id", None): ([
PathNode(name='form'),
PathNode(name='case'),
PathNode(name='create'),
PathNode(name='owner_id')
def test_mismatched_value_type(self):
doc = {'external_blobs': {}, 'photo': "this clearly isn't a photo"}
result = self.column.get_value('my-domain',
"this clearly isn't a photo", doc,
[PathNode(name='form')])
self.assertEqual(result, "this clearly isn't a photo")
def test_stock_conversion(self, _):
instance, _ = self._convert_case_export('stock')
table = instance.get_table(MAIN_TABLE)
path = [PathNode(name='stock')]
index, column = table.get_column(path, 'ExportItem', None)
self.assertTrue(column.selected)
def test_multiple_write_export_instance_calls(self):
"""
Confirm that calling _write_export_instance() multiple times
(as part of a bulk export) works as expected.
"""
export_instances = [
FormExportInstance(
# export_format=Format.JSON,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Q3",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q3')],
),
selected=True,
),
]
),
]
),
FormExportInstance(
# export_format=Format.JSON,
tables=[
TableConfiguration(
label="My other table",
selected=True,
path=[PathNode(name="form", is_repeat=False), PathNode(name="q2", is_repeat=False)],
columns=[
ExportColumn(
label="Q4",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4')],
),
selected=True,
),
]
)
]
)
]
writer = _Writer(get_writer(Format.JSON))
with writer.open(export_instances):
_write_export_instance(writer, export_instances[0], self.docs)
_write_export_instance(writer, export_instances[1], self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
u'Export1-My table': {
u'headers': [u'Q3'],
u'rows': [[u'baz'], [u'bop']],
},
u'Export2-My other table': {
u'headers': [u'Q4'],
u'rows': [[u'bar'], [u'boop']],
}
}
)
def setUpClass(cls):
super(TestConvertSavedExportSchemaToFormExportInstance,
cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='question1')
],
label='Question 1 Not updated',
last_occurrences={cls.app_id: 3},
),
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='deid_id')
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='deid_date')
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 3},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='question2')
],
label='Question 2',
last_occurrences={cls.app_id: 2},
)
],
last_occurrences={cls.app_id: 2},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='repeat_nested', is_repeat=True),
],
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='repeat_nested', is_repeat=True),
PathNode(name='nested'),
],
label='Nested Repeat',
last_occurrences={cls.app_id: 2},
)
],
last_occurrences={cls.app_id: 2},
),
],
)
def test_missing_value(self):
doc = {'external_blobs': {}, 'photo': None}
result = self.column.get_value('my-domain', '1234', doc,
[PathNode(name='form')])
self.assertEqual(result, MISSING_VALUE)
