def test_csv_dataframe_export_to(self):
self._publish_nested_repeats_form()
self._submit_fixture_instance("nested_repeats",
"01",
submission_time=self._submission_time)
self._submit_fixture_instance("nested_repeats",
"02",
submission_time=self._submission_time)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_fixture_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "fixtures",
"nested_repeats", "nested_repeats.csv")
temp_file.close()
fixture, output = '', ''
with open(csv_fixture_path) as f:
fixture = f.read()
with open(temp_file.name) as f:
output = f.read()
os.unlink(temp_file.name)
self.assertEqual(fixture, output)
def to_flat_csv_export(self, path, data, username, id_string, filter_query,
**kwargs):
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
start = kwargs.get('start')
end = kwargs.get('end')
dataview = kwargs.get('dataview')
xform = kwargs.get('xform')
options = kwargs.get('options')
total_records = kwargs.get('total_records')
win_excel_utf8 = options.get('win_excel_utf8') if options else False
csv_builder = CSVDataFrameBuilder(username,
id_string,
filter_query,
self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES,
self.BINARY_SELECT_MULTIPLES,
start,
end,
self.TRUNCATE_GROUP_TITLE,
xform,
self.INCLUDE_LABELS,
self.INCLUDE_LABELS_ONLY,
self.INCLUDE_IMAGES,
self.INCLUDE_HXL,
win_excel_utf8=win_excel_utf8,
total_records=total_records)
csv_builder.export_to(path, dataview=dataview)
def test_csv_export(self):
self._publish_single_level_repeat_form()
# submit 7 instances
for i in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for i in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
def test_csv_export(self):
"""
Test CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
def test_windows_excel_compatible_csv_export(self):
"""
Test window excel compatible CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False,
win_excel_utf8=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
self.assertEqual(b'\xef\xbb\xbfname', header[0].encode('utf-8'))
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_export(self):
self._publish_single_level_repeat_form()
# submit 7 instances
for i in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for i in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(
len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def to_flat_csv_export(self, path, data, username, id_string,
filter_query, **kwargs):
"""
Generates a flattened CSV file for submitted data.
"""
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
start = kwargs.get('start')
end = kwargs.get('end')
dataview = kwargs.get('dataview')
xform = kwargs.get('xform')
options = kwargs.get('options')
total_records = kwargs.get('total_records')
win_excel_utf8 = options.get('win_excel_utf8') if options else False
index_tags = options.get(REPEAT_INDEX_TAGS, self.REPEAT_INDEX_TAGS)
show_choice_labels = options.get('show_choice_labels', False)
language = options.get('language')
csv_builder = CSVDataFrameBuilder(
username, id_string, filter_query, self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES,
start, end, self.TRUNCATE_GROUP_TITLE, xform,
self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, self.INCLUDE_IMAGES,
self.INCLUDE_HXL, win_excel_utf8=win_excel_utf8,
total_records=total_records, index_tags=index_tags,
value_select_multiples=self.VALUE_SELECT_MULTIPLES,
show_choice_labels=show_choice_labels, language=language)
csv_builder.export_to(path, dataview=dataview)
def test_csv_export_with_labels(self):
"""
Test CSV export with labels.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_labels=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received'
]
self.assertEqual(expected_header, header)
labels = csv_reader.next()
self.assertEqual(len(labels),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_labels = [
'Name', 'age', 'Do you have kids?', 'Kids Name', 'Kids Age',
'Kids Name', 'Kids Age', '5. Record your GPS coordinates.',
'_gps_latitude', '_gps_longitude', '_gps_altitude',
'_gps_precision', 'web_browsers/Mozilla Firefox',
'web_browsers/Google Chrome', 'web_browsers/Internet Explorer',
'web_browsers/Safari', 'instanceID', '_id', '_uuid',
'_submission_time', '_tags', '_notes', '_version', '_duration',
'_submitted_by', '_total_media', '_media_count',
'_media_all_received'
]
self.assertEqual(expected_labels, labels)
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def to_flat_csv_export(
self, path, data, username, id_string, filter_query,
start=None, end=None):
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
csv_builder = CSVDataFrameBuilder(
username, id_string, filter_query, self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES,
start, end, self.TRUNCATE_GROUP_TITLE
)
csv_builder.export_to(path)
def test_csv_export_extra_columns(self):
"""
Test CSV export EXTRA_COLUMNS
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_labels=True)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received', '_xform_id'
]
self.assertEqual(expected_header, header)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_labels=True)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
self.assertEqual(expected_header, header)
csv_file.close()
os.unlink(temp_file.name)
def to_flat_csv_export(self,
path,
data,
username,
id_string,
filter_query,
start=None,
end=None):
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
csv_builder = CSVDataFrameBuilder(username, id_string, filter_query,
self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES,
self.BINARY_SELECT_MULTIPLES, start,
end, self.TRUNCATE_GROUP_TITLE)
csv_builder.export_to(path)
def test_windows_excel_compatible_csv_export(self):
"""
Test window excel compatible CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False,
win_excel_utf8=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_header = [
'\xef\xbb\xbfname', '\xef\xbb\xbfage', '\xef\xbb\xbfhas_kids',
'\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age',
'\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfgps',
'\xef\xbb\xbf_gps_latitude', '\xef\xbb\xbf_gps_longitude',
'\xef\xbb\xbf_gps_altitude', '\xef\xbb\xbf_gps_precision',
'\xef\xbb\xbfweb_browsers/firefox',
'\xef\xbb\xbfweb_browsers/chrome', '\xef\xbb\xbfweb_browsers/ie',
'\xef\xbb\xbfweb_browsers/safari', '\xef\xbb\xbfinstanceID',
'\xef\xbb\xbf_id', '\xef\xbb\xbf_uuid',
'\xef\xbb\xbf_submission_time', '\xef\xbb\xbf_tags',
'\xef\xbb\xbf_notes', '\xef\xbb\xbf_version',
'\xef\xbb\xbf_duration', '\xef\xbb\xbf_submitted_by',
'\xef\xbb\xbf_total_media', '\xef\xbb\xbf_media_count',
'\xef\xbb\xbf_media_all_received'
]
self.assertEqual(expected_header, header)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_export_remove_group_name(self):
"""
Test CSV export with remove_group_name option.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received'
]
self.assertEqual(expected_header, header)
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_dataframe_export_to(self):
self._publish_nested_repeats_form()
self._submit_fixture_instance(
"nested_repeats", "01", submission_time=self._submission_time)
self._submit_fixture_instance(
"nested_repeats", "02", submission_time=self._submission_time)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_fixture_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"fixtures", "nested_repeats", "nested_repeats.csv"
)
temp_file.close()
fixture, output = '', ''
with open(csv_fixture_path) as f:
fixture = f.read()
with open(temp_file.name) as f:
output = f.read()
os.unlink(temp_file.name)
self.assertEqual(fixture, output)
