def test_empty_table_label(self):
export_instance = FormExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
split_multiselects=True,
tables=[
TableConfiguration(label="",
selected=True,
path=[],
columns=[
ExportColumn(label="Q1",
item=ScalarItem(path=[
PathNode(name='form'),
PathNode(name='q1')
], ),
selected=True),
])
])
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(json.loads(export.read()), {
u'Sheet1': {
u'headers': [u'Q1'],
u'rows': [[u'foo'], [u'bip']],
}
})
def test_multi_table_order(self):
tables = [
TableConfiguration(label="My table {}".format(i),
selected=True,
path=[],
columns=[
ExportColumn(
label="Q{}".format(i),
item=ScalarItem(path=[
PathNode(name='form'),
PathNode(name='q{}'.format(i))
], ),
selected=True,
),
]) for i in range(10)
]
export_instance = FormExportInstance(export_format=Format.HTML,
tables=tables)
writer = get_export_writer([export_instance])
docs = [{
'domain': 'my-domain',
'_id': '1234',
"form": {'q{}'.format(i): 'value {}'.format(i)
for i in range(10)}
}]
with writer.open([export_instance]):
write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
exported_tables = [
table for table in re.findall('<table>', export.read())
]
expected_tables = [t.label for t in tables]
self.assertEqual(len(expected_tables), len(exported_tables))
def assert_instance_gives_results(docs, export_instance, expected_result):
with TransientTempfile() as temp_path:
writer = get_export_writer([export_instance], temp_path)
with writer.open([export_instance]):
write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
assert json.loads(export.read()) == expected_result
def assert_instance_gives_results(docs, export_instance, expected_result):
with TransientTempfile() as temp_path:
writer = get_export_writer([export_instance], temp_path)
with writer.open([export_instance]):
write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
assert json.loads(export.read()) == expected_result
def _get_export_file_path(export_instance, docs, progress_tracker=None):
export_instances = [export_instance]
# Multiprocess exports sometimes intentionally keep the tempfile,
# so TransientTempfile isn't appropriate here
fd, temp_path = tempfile.mkstemp()
os.close(fd)
writer = get_export_writer(export_instances,
temp_path,
allow_pagination=False)
with writer.open(export_instances):
write_export_instance(writer, export_instance, docs, progress_tracker)
return writer.path
def test_multi_table(self):
export_instance = FormExportInstance(
export_format=Format.JSON,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Q3",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q3')],
),
selected=True,
),
]
),
TableConfiguration(
label="My other table",
selected=True,
path=[PathNode(name='form', is_repeat=False), PathNode(name="q2", is_repeat=False)],
columns=[
ExportColumn(
label="Q4",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4')],
),
selected=True,
),
]
)
]
)
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table': {
'headers': ['Q3'],
'rows': [['baz'], ['bop']],
},
'My other table': {
'headers': ['Q4'],
'rows': [['bar'], ['boop']],
}
}
)
def test_case_name_transform(self):
docs = [
{
'domain': 'my-domain',
'_id': '1234',
"form": {
"caseid": "robin",
},
},
{
'domain': 'my-domain',
'_id': '1234',
"form": {
"caseid": "i-do-not-exist",
},
}
]
export_instance = FormExportInstance(
export_format=Format.JSON,
tables=[
TableConfiguration(
label="My table",
selected=True,
columns=[
ExportColumn(
label="case_name",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='caseid')],
transform=CASE_NAME_TRANSFORM,
),
selected=True
),
]
)
]
)
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table': {
'headers': ['case_name'],
'rows': [['batman'], [MISSING_VALUE]],
}
}
)
def _generate_incremental_export(incremental_export, last_doc_date=None):
export_instance = incremental_export.export_instance
export_instance.export_format = Format.UNZIPPED_CSV # force to unzipped CSV
# Remove the date period from the ExportInstance, since this is added automatically by Daily Saved exports
export_instance.filters.date_period = None
filters = export_instance.get_filters()
if last_doc_date:
filters.append(ServerModifiedOnRangeFilter(gt=last_doc_date))
class LastDocTracker:
def __init__(self, doc_iterator):
self.doc_iterator = doc_iterator
self.last_doc = None
self.doc_count = 0
def __iter__(self):
for doc in self.doc_iterator:
self.last_doc = doc
self.doc_count += 1
yield doc
with TransientTempfile() as temp_path, metrics_track_errors(
'generate_incremental_exports'):
writer = get_export_writer([export_instance],
temp_path,
allow_pagination=False)
with writer.open([export_instance]):
query = get_export_query(export_instance, filters)
query = query.sort('server_modified_on'
) # reset sort to this instead of opened_on
docs = LastDocTracker(query.run().hits)
write_export_instance(writer, export_instance, docs)
export_file = ExportFile(writer.path, writer.format)
if docs.doc_count <= 0:
return
new_checkpoint = incremental_export.checkpoint(
docs.doc_count, docs.last_doc.get('server_modified_on'))
with export_file as file_:
db = get_blob_db()
db.put(file_,
domain=incremental_export.domain,
parent_id=new_checkpoint.blob_parent_id,
type_code=CODES.data_export,
key=str(new_checkpoint.blob_key),
timeout=24 * 60)
return new_checkpoint
def test_simple_table(self):
"""
Confirm that some simple documents and a simple FormExportInstance
are writtern with _write_export_file() correctly
"""
export_instance = FormExportInstance(
export_format=Format.JSON,
tables=[
TableConfiguration(
label="My table",
selected=True,
columns=[
ExportColumn(
label="Q3",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q3')],
),
selected=True
),
ExportColumn(
label="Q1",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q1')],
),
selected=True
),
]
)
]
)
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table': {
'headers': ['Q3', 'Q1'],
'rows': [['baz', 'foo'], ['bop', 'bip']],
}
}
)
def test_empty_location(self):
export_instance = FormExportInstance(
export_format=Format.JSON,
tables=[
TableConfiguration(
label="My table",
selected=True,
columns=[
ExportColumn(
label="location",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='meta'), PathNode(name='location')],
),
selected=True
),
]
)
]
)
docs = [
{
'domain': 'my-domain',
'_id': '1234',
'form': {
'meta': {
'location': {'xmlns': 'abc'},
}
}
}
]
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table': {
'headers': ['location'],
'rows': [[EMPTY_VALUE]],
}
}
)
def test_array_data_in_scalar_question(self):
'''
This test ensures that when a question id has array data
that we return still return a string for scalar data.
This happens rarely
'''
doc = {
'domain': 'my-domain',
'_id': '12345',
"form": {
"array": ["one", "two"],
}
}
export_instance = FormExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
xmlns='xmlns',
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Scalar Array",
item=ScalarItem(path=[PathNode(name='form'), PathNode(name='array')]),
selected=True,
)
]
)]
)
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, [doc])
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table': {
'headers': ['Scalar Array'],
'rows': [['one two']],
}
}
)
def test_paginated_table(self):
export_instance = FormExportInstance(
export_format=Format.JSON,
tables=[
TableConfiguration(
label="My table",
selected=True,
columns=[
ExportColumn(
label="Q3",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q3')],
),
selected=True
),
ExportColumn(
label="Q1",
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q1')],
),
selected=True
),
]
)
]
)
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, self.docs + self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table_000': {
'headers': ['Q3', 'Q1'],
'rows': [['baz', 'foo'], ['bop', 'bip']],
},
'My table_001': {
'headers': ['Q3', 'Q1'],
'rows': [['baz', 'foo'], ['bop', 'bip']],
}
}
)
def test_multi_table_order(self, export_save):
tables = [
TableConfiguration(
label="My table {}".format(i),
selected=True,
path=[],
columns=[
ExportColumn(
label="Q{}".format(i),
item=ScalarItem(
path=[PathNode(name='form'), PathNode(name='q{}'.format(i))],
),
selected=True,
),
]
)
for i in range(10)
]
export_instance = FormExportInstance(
export_format=Format.HTML,
tables=tables
)
docs = [
{
'domain': 'my-domain',
'_id': '1234',
"form": {'q{}'.format(i): 'value {}'.format(i) for i in range(10)}
}
]
with TransientTempfile() as temp_path:
writer = get_export_writer([export_instance], temp_path)
with writer.open([export_instance]):
write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
exported_tables = [table for table in re.findall(b'<table>', export.read())]
expected_tables = [t.label for t in tables]
self.assertEqual(len(expected_tables), len(exported_tables))
self.assertTrue(export_save.called)
def test_split_questions_false(self):
"""Ensure multiselects are not split when `split_multiselects` is set to False"""
export_instance = FormExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
split_multiselects=False,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
SplitExportColumn(
label="MC",
item=MultipleChoiceItem(
path=[PathNode(name='form'), PathNode(name='mc')],
options=[
Option(value='one'),
Option(value='two'),
]
),
selected=True,
)
]
)]
)
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table': {
'headers': ['MC'],
'rows': [['two extra'], ['one two']],
}
}
)
def test_transform_dates(self):
"""Ensure dates are transformed for excel when `transform_dates` is set to True"""
export_instance = FormExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
transform_dates=True,
tables=[
TableConfiguration(label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Date",
item=MultipleChoiceItem(path=[
PathNode(name='form'),
PathNode(name='date')
], ),
selected=True,
)
])
])
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, self.docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()), {
u'My table': {
u'headers': [u'Date'],
u'rows': [[MISSING_VALUE],
[
couch_to_excel_datetime(
'2015-07-22T14:16:49.584880Z', None)
]],
}
})
def get_export_file(export_instance, docs, progress_tracker=None):
export_instances = [export_instance]
writer = get_export_writer(export_instances, allow_pagination=False)
with writer.open(export_instances):
write_export_instance(writer, export_instance, docs, progress_tracker)
return ExportFile(writer.path, writer.format)
def test_form_stock_columns(self):
"""Ensure that we can export stock properties in a form export"""
docs = [{
'_id': 'simone-biles',
'domain': DOMAIN,
'form': {
'balance': [
{
'@type': 'question-id',
'entry': {
'@quantity': '2',
}
}, {
'@type': 'other-question-id',
'entry': {
'@quantity': '3',
}
}]
},
}, {
'_id': 'sam-mikulak',
'domain': DOMAIN,
'form': {
'balance': {
'@type': 'question-id',
'entry': {
'@quantity': '2',
}
},
},
}, {
'_id': 'kerri-walsh',
'domain': DOMAIN,
'form': {
'balance': {
'@type': 'other-question-id',
'entry': {
'@quantity': '2',
}
},
},
}, {
'_id': 'april-ross',
'domain': DOMAIN,
'form': {},
}]
export_instance = FormExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
StockFormExportColumn(
label="StockItem @type",
item=StockItem(
path=[
PathNode(name='form'),
PathNode(name='balance:question-id'),
PathNode(name='@type'),
],
),
selected=True,
),
StockFormExportColumn(
label="StockItem @quantity",
item=StockItem(
path=[
PathNode(name='form'),
PathNode(name='balance:question-id'),
PathNode(name='entry'),
PathNode(name='@quantity'),
],
),
selected=True,
),
]
)]
)
writer = get_export_writer([export_instance])
with writer.open([export_instance]):
write_export_instance(writer, export_instance, docs)
with ExportFile(writer.path, writer.format) as export:
self.assertEqual(
json.loads(export.read()),
{
'My table': {
'headers': ['StockItem @type', 'StockItem @quantity'],
'rows': [
['question-id', '2'],
['question-id', '2'],
[MISSING_VALUE, MISSING_VALUE],
[MISSING_VALUE, MISSING_VALUE],
],
}
}
)
