def test_csv_export(self): self._publish_single_level_repeat_form() # submit 7 instances for i in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for i in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close()
def _csv_data_for_dataframe(self): csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) # pylint: disable=protected-access cursor = csv_df_builder._query_data() return [d for d in csv_df_builder._format_for_dataframe(cursor)]
def test_windows_excel_compatible_csv_export(self): """ Test window excel compatible CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False, win_excel_utf8=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) self.assertEqual(b'\xef\xbb\xbfname', header[0].encode('utf-8')) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_export(self): self._publish_single_level_repeat_form() # submit 7 instances for i in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for i in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual( len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_export(self): """ Test CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close()
def test_split_select_multiples_values(self): """ Test select multiples choices are split and their values as the data. """ self._publish_nested_repeats_form() self._submit_fixture_instance("nested_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) # pylint: disable=protected-access cursor = [k for k in csv_df_builder._query_data()] record = cursor[0] select_multiples = \ CSVDataFrameBuilder._collect_select_multiples(self.xform) result = CSVDataFrameBuilder._split_select_multiples( record, select_multiples, value_select_multiples=True) expected_result = { u'web_browsers/ie': u'ie', u'web_browsers/safari': u'safari', u'web_browsers/firefox': None, u'web_browsers/chrome': None } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in list(result) if key in list(expected_result)]) self.assertEqual(expected_result, result)
def test_csv_export_with_labels(self): """ Test CSV export with labels. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_labels=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_header, header) labels = csv_reader.next() self.assertEqual(len(labels), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_labels = [ 'Name', 'age', 'Do you have kids?', 'Kids Name', 'Kids Age', 'Kids Name', 'Kids Age', '5. Record your GPS coordinates.', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/Mozilla Firefox', 'web_browsers/Google Chrome', 'web_browsers/Internet Explorer', 'web_browsers/Safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_labels, labels) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_no_split_select_multiples(self, mock_query_data): """ Test selct multiples are not split within repeats. """ md_xform = """ | survey | | | type | name | label | | | text | name | Name | | | integer | age | Age | | | begin repeat | browser_use | Browser Use | | | integer | year | Year | | | select_multiple browsers | browsers | Browsers | | | end repeat | | | | choices | | | list name | name | label | | | browsers | firefox | Firefox | | | browsers | chrome | Chrome | | | browsers | ie | Internet Explorer | | | browsers | safari | Safari | """ xform = self._publish_markdown(md_xform, self.user, id_string='b') data = [{ 'name': 'Tom', 'age': 23, 'browser_use': [{ 'browser_use/year': '2010', 'browser_use/browsers': 'firefox safari' }, { 'browser_use/year': '2011', 'browser_use/browsers': 'firefox chrome' }] }] # yapf: disable mock_query_data.return_value = data csv_df_builder = CSVDataFrameBuilder(self.user.username, xform.id_string, split_select_multiples=False, include_images=False) # pylint: disable=protected-access cursor = [row for row in csv_df_builder._query_data()] result = [k for k in csv_df_builder._format_for_dataframe(cursor)] expected_result = [{ 'name': 'Tom', 'age': 23, 'browser_use[1]/year': '2010', 'browser_use[1]/browsers': 'firefox safari', 'browser_use[2]/year': '2011', 'browser_use[2]/browsers': 'firefox chrome' }] self.maxDiff = None self.assertEqual(expected_result, result)
def test_windows_excel_compatible_csv_export(self): """ Test window excel compatible CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False, win_excel_utf8=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_header = [ '\xef\xbb\xbfname', '\xef\xbb\xbfage', '\xef\xbb\xbfhas_kids', '\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfgps', '\xef\xbb\xbf_gps_latitude', '\xef\xbb\xbf_gps_longitude', '\xef\xbb\xbf_gps_altitude', '\xef\xbb\xbf_gps_precision', '\xef\xbb\xbfweb_browsers/firefox', '\xef\xbb\xbfweb_browsers/chrome', '\xef\xbb\xbfweb_browsers/ie', '\xef\xbb\xbfweb_browsers/safari', '\xef\xbb\xbfinstanceID', '\xef\xbb\xbf_id', '\xef\xbb\xbf_uuid', '\xef\xbb\xbf_submission_time', '\xef\xbb\xbf_tags', '\xef\xbb\xbf_notes', '\xef\xbb\xbf_version', '\xef\xbb\xbf_duration', '\xef\xbb\xbf_submitted_by', '\xef\xbb\xbf_total_media', '\xef\xbb\xbf_media_count', '\xef\xbb\xbf_media_all_received' ] self.assertEqual(expected_header, header) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_export_remove_group_name(self): """ Test CSV export with remove_group_name option. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_header, header) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_split_select_multiples(self): self._publish_nested_repeats_form() self._submit_fixture_instance("nested_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) cursor = [k for k in csv_df_builder._query_data()] record = cursor[0] select_multiples = \ CSVDataFrameBuilder._collect_select_multiples(self.xform) result = CSVDataFrameBuilder._split_select_multiples( record, select_multiples) expected_result = { u'web_browsers/ie': True, u'web_browsers/safari': True, u'web_browsers/firefox': False, u'web_browsers/chrome': False } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, binary_select_multiples=True) result = csv_df_builder._split_select_multiples( record, select_multiples) expected_result = { u'web_browsers/ie': 1, u'web_browsers/safari': 1, u'web_browsers/firefox': 0, u'web_browsers/chrome': 0 } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result)
def test_split_select_multiples(self): self._publish_nested_repeats_form() dd = self.xform.data_dictionary() self._submit_fixture_instance("nested_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) cursor = [k for k in csv_df_builder._query_data()] record = cursor[0] select_multiples = CSVDataFrameBuilder._collect_select_multiples(dd) result = CSVDataFrameBuilder._split_select_multiples(record, select_multiples) expected_result = { u'web_browsers/ie': True, u'web_browsers/safari': True, u'web_browsers/firefox': False, u'web_browsers/chrome': False } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, binary_select_multiples=True) result = csv_df_builder._split_select_multiples(record, select_multiples) expected_result = { u'web_browsers/ie': 1, u'web_browsers/safari': 1, u'web_browsers/firefox': 0, u'web_browsers/chrome': 0 } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result)
def _csv_data_for_dataframe(self): csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) cursor = csv_df_builder._query_data() return csv_df_builder._format_for_dataframe(cursor)
def test_index_tag_replacement(self): """ Test that the default index tags are correctly replaced by an underscore """ self._publish_xls_fixture_set_xform("groups_in_repeats") self._submit_fixture_instance("groups_in_repeats", "01") self.xform.get_keys() csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False, index_tags=('_', '_')) cursor = csv_df_builder._query_data() result = [d for d in csv_df_builder._format_for_dataframe(cursor)][0] # remove dynamic fields ignore_list = [ '_uuid', 'meta/instanceID', 'formhub/uuid', '_submission_time', '_id', '_bamboo_dataset_id' ] for item in ignore_list: result.pop(item) expected_result = { u'_xform_id_string': u'groups_in_repeats', u'_xform_id': self.xform.pk, u'_status': u'submitted_via_web', u'_tags': u'', u'_notes': u'', u'_version': self.xform.version, u"_submitted_by": u'bob', u'name': u'Abe', u'age': 88, u'has_children': u'1', u'children_1_/childs_info/name': u'Cain', u'children_2_/childs_info/name': u'Abel', u'children_1_/childs_info/age': 56, u'children_2_/childs_info/age': 48, u'children_1_/immunization/immunization_received/polio_1': True, u'children_1_/immunization/immunization_received/polio_2': False, u'children_2_/immunization/immunization_received/polio_1': True, u'children_2_/immunization/immunization_received/polio_2': True, u'web_browsers/chrome': True, u'web_browsers/firefox': False, u'web_browsers/ie': False, u'web_browsers/safari': False, u'gps': u'-1.2626156 36.7923571 0.0 30.0', u'_geolocation': [-1.2626156, 36.7923571], u'_duration': '', u'_edited': False, u'_gps_latitude': u'-1.2626156', u'_gps_longitude': u'36.7923571', u'_gps_altitude': u'0.0', u'_gps_precision': u'30.0', u'_attachments': [], u'_total_media': 0, u'_media_count': 0, u'_media_all_received': True } self.maxDiff = None self.assertEqual(expected_result, result)
def _csv_data_for_dataframe(self): csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) cursor = csv_df_builder._query_data() return csv_df_builder._format_for_dataframe(cursor)