def test_empty_table_label(self): export_instance = FormExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, split_multiselects=True, tables=[ TableConfiguration(label="", selected=True, path=[], columns=[ ExportColumn(label="Q1", item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q1') ], ), selected=True), ]) ]) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual(json.loads(export.read()), { u'Sheet1': { u'headers': [u'Q1'], u'rows': [[u'foo'], [u'bip']], } })
def test_multi_table_order(self): tables = [ TableConfiguration(label="My table {}".format(i), selected=True, path=[], columns=[ ExportColumn( label="Q{}".format(i), item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q{}'.format(i)) ], ), selected=True, ), ]) for i in range(10) ] export_instance = FormExportInstance(export_format=Format.HTML, tables=tables) writer = _get_writer([export_instance]) docs = [{ 'domain': 'my-domain', '_id': '1234', "form": {'q{}'.format(i): 'value {}'.format(i) for i in range(10)} }] with writer.open([export_instance]): _write_export_instance(writer, export_instance, docs) with ExportFile(writer.path, writer.format) as export: exported_tables = [ table for table in re.findall('<h2>(.*)</h2>', export.read()) ] expected_tables = [t.label for t in tables] self.assertEqual(expected_tables, exported_tables)
def assert_instance_gives_results(docs, export_instance, expected_result): with TransientTempfile() as temp_path: writer = get_export_writer([export_instance], temp_path) with writer.open([export_instance]): write_export_instance(writer, export_instance, docs) with ExportFile(writer.path, writer.format) as export: assert json.loads(export.read()) == expected_result
def test_multi_table(self): export_instance = FormExportInstance( export_format=Format.JSON, tables=[ TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Q3", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q3')], ), selected=True, ), ] ), TableConfiguration( label="My other table", selected=True, path=[PathNode(name='form', is_repeat=False), PathNode(name="q2", is_repeat=False)], columns=[ ExportColumn( label="Q4", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4')], ), selected=True, ), ] ) ] ) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [u'Q3'], u'rows': [[u'baz'], [u'bop']], }, u'My other table': { u'headers': [u'Q4'], u'rows': [[u'bar'], [u'boop']], } } )
def test_case_name_transform(self): docs = [ { 'domain': 'my-domain', '_id': '1234', "form": { "caseid": "robin", }, }, { 'domain': 'my-domain', '_id': '1234', "form": { "caseid": "i-do-not-exist", }, } ] export_instance = FormExportInstance( export_format=Format.JSON, tables=[ TableConfiguration( label="My table", selected=True, columns=[ ExportColumn( label="case_name", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='caseid')], transform=CASE_NAME_TRANSFORM, ), selected=True ), ] ) ] ) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [u'case_name'], u'rows': [[u'batman'], [MISSING_VALUE]], } } )
def _generate_incremental_export(incremental_export, last_doc_date=None): export_instance = incremental_export.export_instance export_instance.export_format = Format.UNZIPPED_CSV # force to unzipped CSV # Remove the date period from the ExportInstance, since this is added automatically by Daily Saved exports export_instance.filters.date_period = None filters = export_instance.get_filters() if last_doc_date: filters.append(ServerModifiedOnRangeFilter(gt=last_doc_date)) class LastDocTracker: def __init__(self, doc_iterator): self.doc_iterator = doc_iterator self.last_doc = None self.doc_count = 0 def __iter__(self): for doc in self.doc_iterator: self.last_doc = doc self.doc_count += 1 yield doc with TransientTempfile() as temp_path, metrics_track_errors( 'generate_incremental_exports'): writer = get_export_writer([export_instance], temp_path, allow_pagination=False) with writer.open([export_instance]): query = get_export_query(export_instance, filters) query = query.sort('server_modified_on' ) # reset sort to this instead of opened_on docs = LastDocTracker(query.run().hits) write_export_instance(writer, export_instance, docs) export_file = ExportFile(writer.path, writer.format) if docs.doc_count <= 0: return new_checkpoint = incremental_export.checkpoint( docs.doc_count, docs.last_doc.get('server_modified_on')) with export_file as file_: db = get_blob_db() db.put(file_, domain=incremental_export.domain, parent_id=new_checkpoint.blob_parent_id, type_code=CODES.data_export, key=str(new_checkpoint.blob_key), timeout=24 * 60) return new_checkpoint
def test_simple_table(self): """ Confirm that some simple documents and a simple FormExportInstance are writtern with _write_export_file() correctly """ export_instance = FormExportInstance( export_format=Format.JSON, tables=[ TableConfiguration( label="My table", selected=True, columns=[ ExportColumn( label="Q3", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q3')], ), selected=True ), ExportColumn( label="Q1", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q1')], ), selected=True ), ] ) ] ) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [u'Q3', u'Q1'], u'rows': [[u'baz', u'foo'], [u'bop', u'bip']], } } )
def test_empty_location(self): export_instance = FormExportInstance( export_format=Format.JSON, tables=[ TableConfiguration( label="My table", selected=True, columns=[ ExportColumn( label="location", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='meta'), PathNode(name='location')], ), selected=True ), ] ) ] ) docs = [ { 'domain': 'my-domain', '_id': '1234', 'form': { 'meta': { 'location': {'xmlns': 'abc'}, } } } ] writer = get_export_writer([export_instance]) with writer.open([export_instance]): write_export_instance(writer, export_instance, docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { 'My table': { 'headers': ['location'], 'rows': [[EMPTY_VALUE]], } } )
def test_array_data_in_scalar_question(self): ''' This test ensures that when a question id has array data that we return still return a string for scalar data. This happens rarely ''' doc = { 'domain': 'my-domain', '_id': '12345', "form": { "array": ["one", "two"], } } export_instance = FormExportInstance( export_format=Format.JSON, domain=DOMAIN, xmlns='xmlns', tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Scalar Array", item=ScalarItem(path=[PathNode(name='form'), PathNode(name='array')]), selected=True, ) ] )] ) writer = get_export_writer([export_instance]) with writer.open([export_instance]): write_export_instance(writer, export_instance, [doc]) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { 'My table': { 'headers': ['Scalar Array'], 'rows': [['one two']], } } )
def test_paginated_table(self): export_instance = FormExportInstance( export_format=Format.JSON, tables=[ TableConfiguration( label="My table", selected=True, columns=[ ExportColumn( label="Q3", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q3')], ), selected=True ), ExportColumn( label="Q1", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q1')], ), selected=True ), ] ) ] ) writer = get_export_writer([export_instance]) with writer.open([export_instance]): write_export_instance(writer, export_instance, self.docs + self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { 'My table_000': { 'headers': ['Q3', 'Q1'], 'rows': [['baz', 'foo'], ['bop', 'bip']], }, 'My table_001': { 'headers': ['Q3', 'Q1'], 'rows': [['baz', 'foo'], ['bop', 'bip']], } } )
def test_split_questions_false(self): """Ensure multiselects are not split when `split_multiselects` is set to False""" export_instance = FormExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, split_multiselects=False, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ SplitExportColumn( label="MC", item=MultipleChoiceItem( path=[PathNode(name='form'), PathNode(name='mc')], options=[ Option(value='one'), Option(value='two'), ] ), selected=True, ) ] )] ) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [u'MC'], u'rows': [['two extra'], ['one two']], } } )
def test_transform_dates(self): """Ensure dates are transformed for excel when `transform_dates` is set to True""" export_instance = FormExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, transform_dates=True, tables=[ TableConfiguration(label="My table", selected=True, path=[], columns=[ ExportColumn( label="Date", item=MultipleChoiceItem(path=[ PathNode(name='form'), PathNode(name='date') ], ), selected=True, ) ]) ]) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [u'Date'], u'rows': [[MISSING_VALUE], [ couch_to_excel_datetime( '2015-07-22T14:16:49.584880Z', None) ]], } })
def test_multiple_write_export_instance_calls(self): """ Confirm that calling _write_export_instance() multiple times (as part of a bulk export) works as expected. """ export_instances = [ FormExportInstance( # export_format=Format.JSON, tables=[ TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Q3", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q3')], ), selected=True, ), ] ), ] ), FormExportInstance( # export_format=Format.JSON, tables=[ TableConfiguration( label="My other table", selected=True, path=[PathNode(name="form", is_repeat=False), PathNode(name="q2", is_repeat=False)], columns=[ ExportColumn( label="Q4", item=ScalarItem( path=[PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4')], ), selected=True, ), ] ) ] ) ] writer = _Writer(get_writer(Format.JSON)) with writer.open(export_instances): _write_export_instance(writer, export_instances[0], self.docs) _write_export_instance(writer, export_instances[1], self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'Export1-My table': { u'headers': [u'Q3'], u'rows': [[u'baz'], [u'bop']], }, u'Export2-My other table': { u'headers': [u'Q4'], u'rows': [[u'bar'], [u'boop']], } } )
def test_form_stock_columns(self): """Ensure that we can export stock properties in a form export""" docs = [{ '_id': 'simone-biles', 'domain': DOMAIN, 'form': { 'balance': [ { '@type': 'question-id', 'entry': { '@quantity': '2', } }, { '@type': 'other-question-id', 'entry': { '@quantity': '3', } }] }, }, { '_id': 'sam-mikulak', 'domain': DOMAIN, 'form': { 'balance': { '@type': 'question-id', 'entry': { '@quantity': '2', } }, }, }, { '_id': 'kerri-walsh', 'domain': DOMAIN, 'form': { 'balance': { '@type': 'other-question-id', 'entry': { '@quantity': '2', } }, }, }, { '_id': 'april-ross', 'domain': DOMAIN, 'form': {}, }] export_instance = FormExportInstance( export_format=Format.JSON, domain=DOMAIN, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ StockFormExportColumn( label="StockItem @type", item=StockItem( path=[ PathNode(name='form'), PathNode(name='balance:question-id'), PathNode(name='@type'), ], ), selected=True, ), StockFormExportColumn( label="StockItem @quantity", item=StockItem( path=[ PathNode(name='form'), PathNode(name='balance:question-id'), PathNode(name='entry'), PathNode(name='@quantity'), ], ), selected=True, ), ] )] ) writer = _get_writer([export_instance]) with writer.open([export_instance]): _write_export_instance(writer, export_instance, docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [u'StockItem @type', u'StockItem @quantity'], u'rows': [ ['question-id', '2'], ['question-id', '2'], [MISSING_VALUE, MISSING_VALUE], [MISSING_VALUE, MISSING_VALUE], ], } } )
def test_multiple_write_export_instance_calls(self, export_save): """ Confirm that calling _write_export_instance() multiple times (as part of a bulk export) works as expected. """ export_instances = [ FormExportInstance(tables=[ TableConfiguration(label="My table", selected=True, path=[], columns=[ ExportColumn( label="Q3", item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q3') ], ), selected=True, ), ]), ]), FormExportInstance(tables=[ TableConfiguration(label="My other table", selected=True, path=[ PathNode(name="form", is_repeat=False), PathNode(name="q2", is_repeat=False) ], columns=[ ExportColumn( label="Q4", item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4') ], ), selected=True, ), ]) ]), FormExportInstance(tables=[ TableConfiguration(label="My other table", selected=True, path=[ PathNode(name="form", is_repeat=False), PathNode(name="q2", is_repeat=False) ], columns=[ ExportColumn( label="Q4", item=ScalarItem(path=[ PathNode(name='form'), PathNode(name='q2'), PathNode(name='q4') ], ), selected=True, ), ]) ]) ] with TransientTempfile() as temp_path: writer = _ExportWriter(get_writer(Format.JSON), temp_path) with writer.open(export_instances): write_export_instance(writer, export_instances[0], self.docs) write_export_instance(writer, export_instances[1], self.docs) write_export_instance(writer, export_instances[2], self.docs) with ExportFile(writer.path, writer.format) as export: self.assertEqual( json.loads(export.read()), { 'My table': { 'headers': ['Q3'], 'rows': [['baz'], ['bop']], }, 'Export2-My other table': { 'headers': ['Q4'], 'rows': [['bar'], ['boop']], }, 'Export3-My other table': { 'headers': ['Q4'], 'rows': [['bar'], ['boop']], }, }) self.assertTrue(export_save.called)
def get_export_file(export_instance, docs, progress_tracker=None): export_instances = [export_instance] writer = get_export_writer(export_instances, allow_pagination=False) with writer.open(export_instances): write_export_instance(writer, export_instance, docs, progress_tracker) return ExportFile(writer.path, writer.format)