def test_csv_dataframe_export_to(self): self._publish_nested_repeats_form() self._submit_fixture_instance("nested_repeats", "01", submission_time=self._submission_time) self._submit_fixture_instance("nested_repeats", "02", submission_time=self._submission_time) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_fixture_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "fixtures", "nested_repeats", "nested_repeats.csv") temp_file.close() fixture, output = '', '' with open(csv_fixture_path) as f: fixture = f.read() with open(temp_file.name) as f: output = f.read() os.unlink(temp_file.name) self.assertEqual(fixture, output)
def to_flat_csv_export(self, path, data, username, id_string, filter_query, **kwargs): # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder start = kwargs.get('start') end = kwargs.get('end') dataview = kwargs.get('dataview') xform = kwargs.get('xform') options = kwargs.get('options') total_records = kwargs.get('total_records') win_excel_utf8 = options.get('win_excel_utf8') if options else False csv_builder = CSVDataFrameBuilder(username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE, xform, self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, self.INCLUDE_IMAGES, self.INCLUDE_HXL, win_excel_utf8=win_excel_utf8, total_records=total_records) csv_builder.export_to(path, dataview=dataview)
def test_csv_export(self): self._publish_single_level_repeat_form() # submit 7 instances for i in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for i in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close()
def test_csv_export(self): """ Test CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close()
def test_windows_excel_compatible_csv_export(self): """ Test window excel compatible CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False, win_excel_utf8=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) self.assertEqual(b'\xef\xbb\xbfname', header[0].encode('utf-8')) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_export(self): self._publish_single_level_repeat_form() # submit 7 instances for i in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for i in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual( len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def to_flat_csv_export(self, path, data, username, id_string, filter_query, **kwargs): """ Generates a flattened CSV file for submitted data. """ # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder start = kwargs.get('start') end = kwargs.get('end') dataview = kwargs.get('dataview') xform = kwargs.get('xform') options = kwargs.get('options') total_records = kwargs.get('total_records') win_excel_utf8 = options.get('win_excel_utf8') if options else False index_tags = options.get(REPEAT_INDEX_TAGS, self.REPEAT_INDEX_TAGS) show_choice_labels = options.get('show_choice_labels', False) language = options.get('language') csv_builder = CSVDataFrameBuilder( username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE, xform, self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, self.INCLUDE_IMAGES, self.INCLUDE_HXL, win_excel_utf8=win_excel_utf8, total_records=total_records, index_tags=index_tags, value_select_multiples=self.VALUE_SELECT_MULTIPLES, show_choice_labels=show_choice_labels, language=language) csv_builder.export_to(path, dataview=dataview)
def test_csv_export_with_labels(self): """ Test CSV export with labels. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_labels=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_header, header) labels = csv_reader.next() self.assertEqual(len(labels), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_labels = [ 'Name', 'age', 'Do you have kids?', 'Kids Name', 'Kids Age', 'Kids Name', 'Kids Age', '5. Record your GPS coordinates.', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/Mozilla Firefox', 'web_browsers/Google Chrome', 'web_browsers/Internet Explorer', 'web_browsers/Safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_labels, labels) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def to_flat_csv_export( self, path, data, username, id_string, filter_query, start=None, end=None): # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder csv_builder = CSVDataFrameBuilder( username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE ) csv_builder.export_to(path)
def test_csv_export_extra_columns(self): """ Test CSV export EXTRA_COLUMNS """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_labels=True) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received', '_xform_id' ] self.assertEqual(expected_header, header) # close and delete file csv_file.close() os.unlink(temp_file.name) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_labels=True) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) self.assertEqual(expected_header, header) csv_file.close() os.unlink(temp_file.name)
def to_flat_csv_export(self, path, data, username, id_string, filter_query, start=None, end=None): # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder csv_builder = CSVDataFrameBuilder(username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE) csv_builder.export_to(path)
def test_windows_excel_compatible_csv_export(self): """ Test window excel compatible CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False, win_excel_utf8=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_header = [ '\xef\xbb\xbfname', '\xef\xbb\xbfage', '\xef\xbb\xbfhas_kids', '\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfgps', '\xef\xbb\xbf_gps_latitude', '\xef\xbb\xbf_gps_longitude', '\xef\xbb\xbf_gps_altitude', '\xef\xbb\xbf_gps_precision', '\xef\xbb\xbfweb_browsers/firefox', '\xef\xbb\xbfweb_browsers/chrome', '\xef\xbb\xbfweb_browsers/ie', '\xef\xbb\xbfweb_browsers/safari', '\xef\xbb\xbfinstanceID', '\xef\xbb\xbf_id', '\xef\xbb\xbf_uuid', '\xef\xbb\xbf_submission_time', '\xef\xbb\xbf_tags', '\xef\xbb\xbf_notes', '\xef\xbb\xbf_version', '\xef\xbb\xbf_duration', '\xef\xbb\xbf_submitted_by', '\xef\xbb\xbf_total_media', '\xef\xbb\xbf_media_count', '\xef\xbb\xbf_media_all_received' ] self.assertEqual(expected_header, header) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_export_remove_group_name(self): """ Test CSV export with remove_group_name option. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_header, header) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_dataframe_export_to(self): self._publish_nested_repeats_form() self._submit_fixture_instance( "nested_repeats", "01", submission_time=self._submission_time) self._submit_fixture_instance( "nested_repeats", "02", submission_time=self._submission_time) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_fixture_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "fixtures", "nested_repeats", "nested_repeats.csv" ) temp_file.close() fixture, output = '', '' with open(csv_fixture_path) as f: fixture = f.read() with open(temp_file.name) as f: output = f.read() os.unlink(temp_file.name) self.assertEqual(fixture, output)