def test_csv_export(self):
self._publish_single_level_repeat_form()
# submit 7 instances
for i in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for i in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
cursor = csv_df_builder._query_data()
return [d for d in csv_df_builder._format_for_dataframe(cursor)]
def test_windows_excel_compatible_csv_export(self):
"""
Test window excel compatible CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False,
win_excel_utf8=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
self.assertEqual(b'\xef\xbb\xbfname', header[0].encode('utf-8'))
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_export(self):
self._publish_single_level_repeat_form()
# submit 7 instances
for i in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for i in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(
len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_export(self):
"""
Test CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
def test_split_select_multiples_values(self):
"""
Test select multiples choices are split and their values as the data.
"""
self._publish_nested_repeats_form()
self._submit_fixture_instance("nested_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
cursor = [k for k in csv_df_builder._query_data()]
record = cursor[0]
select_multiples = \
CSVDataFrameBuilder._collect_select_multiples(self.xform)
result = CSVDataFrameBuilder._split_select_multiples(
record, select_multiples, value_select_multiples=True)
expected_result = {
u'web_browsers/ie': u'ie',
u'web_browsers/safari': u'safari',
u'web_browsers/firefox': None,
u'web_browsers/chrome': None
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in list(result)
if key in list(expected_result)])
self.assertEqual(expected_result, result)
def test_csv_export_with_labels(self):
"""
Test CSV export with labels.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_labels=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received'
]
self.assertEqual(expected_header, header)
labels = csv_reader.next()
self.assertEqual(len(labels),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_labels = [
'Name', 'age', 'Do you have kids?', 'Kids Name', 'Kids Age',
'Kids Name', 'Kids Age', '5. Record your GPS coordinates.',
'_gps_latitude', '_gps_longitude', '_gps_altitude',
'_gps_precision', 'web_browsers/Mozilla Firefox',
'web_browsers/Google Chrome', 'web_browsers/Internet Explorer',
'web_browsers/Safari', 'instanceID', '_id', '_uuid',
'_submission_time', '_tags', '_notes', '_version', '_duration',
'_submitted_by', '_total_media', '_media_count',
'_media_all_received'
]
self.assertEqual(expected_labels, labels)
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_no_split_select_multiples(self, mock_query_data):
"""
Test selct multiples are not split within repeats.
"""
md_xform = """
| survey |
| | type | name | label |
| | text | name | Name |
| | integer | age | Age |
| | begin repeat | browser_use | Browser Use |
| | integer | year | Year |
| | select_multiple browsers | browsers | Browsers |
| | end repeat | | |
| choices |
| | list name | name | label |
| | browsers | firefox | Firefox |
| | browsers | chrome | Chrome |
| | browsers | ie | Internet Explorer |
| | browsers | safari | Safari |
"""
xform = self._publish_markdown(md_xform, self.user, id_string='b')
data = [{
'name': 'Tom',
'age': 23,
'browser_use': [{
'browser_use/year': '2010',
'browser_use/browsers': 'firefox safari'
}, {
'browser_use/year': '2011',
'browser_use/browsers': 'firefox chrome'
}]
}] # yapf: disable
mock_query_data.return_value = data
csv_df_builder = CSVDataFrameBuilder(self.user.username,
xform.id_string,
split_select_multiples=False,
include_images=False)
# pylint: disable=protected-access
cursor = [row for row in csv_df_builder._query_data()]
result = [k for k in csv_df_builder._format_for_dataframe(cursor)]
expected_result = [{
'name': 'Tom',
'age': 23,
'browser_use[1]/year': '2010',
'browser_use[1]/browsers': 'firefox safari',
'browser_use[2]/year': '2011',
'browser_use[2]/browsers': 'firefox chrome'
}]
self.maxDiff = None
self.assertEqual(expected_result, result)
def test_windows_excel_compatible_csv_export(self):
"""
Test window excel compatible CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False,
win_excel_utf8=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_header = [
'\xef\xbb\xbfname', '\xef\xbb\xbfage', '\xef\xbb\xbfhas_kids',
'\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age',
'\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfgps',
'\xef\xbb\xbf_gps_latitude', '\xef\xbb\xbf_gps_longitude',
'\xef\xbb\xbf_gps_altitude', '\xef\xbb\xbf_gps_precision',
'\xef\xbb\xbfweb_browsers/firefox',
'\xef\xbb\xbfweb_browsers/chrome', '\xef\xbb\xbfweb_browsers/ie',
'\xef\xbb\xbfweb_browsers/safari', '\xef\xbb\xbfinstanceID',
'\xef\xbb\xbf_id', '\xef\xbb\xbf_uuid',
'\xef\xbb\xbf_submission_time', '\xef\xbb\xbf_tags',
'\xef\xbb\xbf_notes', '\xef\xbb\xbf_version',
'\xef\xbb\xbf_duration', '\xef\xbb\xbf_submitted_by',
'\xef\xbb\xbf_total_media', '\xef\xbb\xbf_media_count',
'\xef\xbb\xbf_media_all_received'
]
self.assertEqual(expected_header, header)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_export_remove_group_name(self):
"""
Test CSV export with remove_group_name option.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received'
]
self.assertEqual(expected_header, header)
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_split_select_multiples(self):
self._publish_nested_repeats_form()
self._submit_fixture_instance("nested_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
cursor = [k for k in csv_df_builder._query_data()]
record = cursor[0]
select_multiples = \
CSVDataFrameBuilder._collect_select_multiples(self.xform)
result = CSVDataFrameBuilder._split_select_multiples(
record, select_multiples)
expected_result = {
u'web_browsers/ie': True,
u'web_browsers/safari': True,
u'web_browsers/firefox': False,
u'web_browsers/chrome': False
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys()
if key in expected_result.keys()])
self.assertEqual(expected_result, result)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
binary_select_multiples=True)
result = csv_df_builder._split_select_multiples(
record, select_multiples)
expected_result = {
u'web_browsers/ie': 1,
u'web_browsers/safari': 1,
u'web_browsers/firefox': 0,
u'web_browsers/chrome': 0
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys()
if key in expected_result.keys()])
self.assertEqual(expected_result, result)
def test_split_select_multiples(self):
self._publish_nested_repeats_form()
dd = self.xform.data_dictionary()
self._submit_fixture_instance("nested_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
cursor = [k for k in csv_df_builder._query_data()]
record = cursor[0]
select_multiples = CSVDataFrameBuilder._collect_select_multiples(dd)
result = CSVDataFrameBuilder._split_select_multiples(record,
select_multiples)
expected_result = {
u'web_browsers/ie': True,
u'web_browsers/safari': True,
u'web_browsers/firefox': False,
u'web_browsers/chrome': False
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys() if key in
expected_result.keys()])
self.assertEqual(expected_result, result)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
binary_select_multiples=True)
result = csv_df_builder._split_select_multiples(record,
select_multiples)
expected_result = {
u'web_browsers/ie': 1,
u'web_browsers/safari': 1,
u'web_browsers/firefox': 0,
u'web_browsers/chrome': 0
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys() if key in
expected_result.keys()])
self.assertEqual(expected_result, result)
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
cursor = csv_df_builder._query_data()
return csv_df_builder._format_for_dataframe(cursor)
def test_index_tag_replacement(self):
"""
Test that the default index tags are correctly replaced by an
underscore
"""
self._publish_xls_fixture_set_xform("groups_in_repeats")
self._submit_fixture_instance("groups_in_repeats", "01")
self.xform.get_keys()
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False,
index_tags=('_', '_'))
cursor = csv_df_builder._query_data()
result = [d for d in csv_df_builder._format_for_dataframe(cursor)][0]
# remove dynamic fields
ignore_list = [
'_uuid', 'meta/instanceID', 'formhub/uuid', '_submission_time',
'_id', '_bamboo_dataset_id'
]
for item in ignore_list:
result.pop(item)
expected_result = {
u'_xform_id_string': u'groups_in_repeats',
u'_xform_id': self.xform.pk,
u'_status': u'submitted_via_web',
u'_tags': u'',
u'_notes': u'',
u'_version': self.xform.version,
u"_submitted_by": u'bob',
u'name': u'Abe',
u'age': 88,
u'has_children': u'1',
u'children_1_/childs_info/name': u'Cain',
u'children_2_/childs_info/name': u'Abel',
u'children_1_/childs_info/age': 56,
u'children_2_/childs_info/age': 48,
u'children_1_/immunization/immunization_received/polio_1': True,
u'children_1_/immunization/immunization_received/polio_2': False,
u'children_2_/immunization/immunization_received/polio_1': True,
u'children_2_/immunization/immunization_received/polio_2': True,
u'web_browsers/chrome': True,
u'web_browsers/firefox': False,
u'web_browsers/ie': False,
u'web_browsers/safari': False,
u'gps': u'-1.2626156 36.7923571 0.0 30.0',
u'_geolocation': [-1.2626156, 36.7923571],
u'_duration': '',
u'_edited': False,
u'_gps_latitude': u'-1.2626156',
u'_gps_longitude': u'36.7923571',
u'_gps_altitude': u'0.0',
u'_gps_precision': u'30.0',
u'_attachments': [],
u'_total_media': 0,
u'_media_count': 0,
u'_media_all_received': True
}
self.maxDiff = None
self.assertEqual(expected_result, result)
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
cursor = csv_df_builder._query_data()
return csv_df_builder._format_for_dataframe(cursor)
