def test_csv_export(self): """ Test CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close()
def _csv_data_for_dataframe(self): csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) # pylint: disable=protected-access cursor = csv_df_builder._query_data() return [d for d in csv_df_builder._format_for_dataframe(cursor)]
def to_flat_csv_export(self, path, data, username, id_string, filter_query, **kwargs): # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder start = kwargs.get('start') end = kwargs.get('end') dataview = kwargs.get('dataview') xform = kwargs.get('xform') options = kwargs.get('options') total_records = kwargs.get('total_records') win_excel_utf8 = options.get('win_excel_utf8') if options else False csv_builder = CSVDataFrameBuilder(username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE, xform, self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, self.INCLUDE_IMAGES, self.INCLUDE_HXL, win_excel_utf8=win_excel_utf8, total_records=total_records) csv_builder.export_to(path, dataview=dataview)
def test_csv_export(self): self._publish_single_level_repeat_form() # submit 7 instances for i in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for i in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual( len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_split_select_multiples_values(self): """ Test select multiples choices are split and their values as the data. """ self._publish_nested_repeats_form() self._submit_fixture_instance("nested_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) # pylint: disable=protected-access cursor = [k for k in csv_df_builder._query_data()] record = cursor[0] select_multiples = \ CSVDataFrameBuilder._collect_select_multiples(self.xform) result = CSVDataFrameBuilder._split_select_multiples( record, select_multiples, value_select_multiples=True) expected_result = { u'web_browsers/ie': u'ie', u'web_browsers/safari': u'safari', u'web_browsers/firefox': None, u'web_browsers/chrome': None } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in list(result) if key in list(expected_result)]) self.assertEqual(expected_result, result)
def test_csv_export(self): self._publish_single_level_repeat_form() # submit 7 instances for i in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for i in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close()
def to_flat_csv_export(self, path, data, username, id_string, filter_query, **kwargs): """ Generates a flattened CSV file for submitted data. """ # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder start = kwargs.get('start') end = kwargs.get('end') dataview = kwargs.get('dataview') xform = kwargs.get('xform') options = kwargs.get('options') total_records = kwargs.get('total_records') win_excel_utf8 = options.get('win_excel_utf8') if options else False index_tags = options.get(REPEAT_INDEX_TAGS, self.REPEAT_INDEX_TAGS) show_choice_labels = options.get('show_choice_labels', False) language = options.get('language') csv_builder = CSVDataFrameBuilder( username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE, xform, self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, self.INCLUDE_IMAGES, self.INCLUDE_HXL, win_excel_utf8=win_excel_utf8, total_records=total_records, index_tags=index_tags, value_select_multiples=self.VALUE_SELECT_MULTIPLES, show_choice_labels=show_choice_labels, language=language) csv_builder.export_to(path, dataview=dataview)
def test_windows_excel_compatible_csv_export(self): """ Test window excel compatible CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False, win_excel_utf8=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) self.assertEqual(b'\xef\xbb\xbfname', header[0].encode('utf-8')) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_dataframe_export_to(self): self._publish_nested_repeats_form() self._submit_fixture_instance("nested_repeats", "01", submission_time=self._submission_time) self._submit_fixture_instance("nested_repeats", "02", submission_time=self._submission_time) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_fixture_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "fixtures", "nested_repeats", "nested_repeats.csv") temp_file.close() fixture, output = '', '' with open(csv_fixture_path) as f: fixture = f.read() with open(temp_file.name) as f: output = f.read() os.unlink(temp_file.name) self.assertEqual(fixture, output)
def test_csv_export_with_labels(self): """ Test CSV export with labels. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_labels=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_header, header) labels = csv_reader.next() self.assertEqual(len(labels), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_labels = [ 'Name', 'age', 'Do you have kids?', 'Kids Name', 'Kids Age', 'Kids Name', 'Kids Age', '5. Record your GPS coordinates.', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/Mozilla Firefox', 'web_browsers/Google Chrome', 'web_browsers/Internet Explorer', 'web_browsers/Safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_labels, labels) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def to_flat_csv_export( self, path, data, username, id_string, filter_query, start=None, end=None): # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder csv_builder = CSVDataFrameBuilder( username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE ) csv_builder.export_to(path)
def test_no_split_select_multiples(self, mock_query_data): """ Test selct multiples are not split within repeats. """ md_xform = """ | survey | | | type | name | label | | | text | name | Name | | | integer | age | Age | | | begin repeat | browser_use | Browser Use | | | integer | year | Year | | | select_multiple browsers | browsers | Browsers | | | end repeat | | | | choices | | | list name | name | label | | | browsers | firefox | Firefox | | | browsers | chrome | Chrome | | | browsers | ie | Internet Explorer | | | browsers | safari | Safari | """ xform = self._publish_markdown(md_xform, self.user, id_string='b') data = [{ 'name': 'Tom', 'age': 23, 'browser_use': [{ 'browser_use/year': '2010', 'browser_use/browsers': 'firefox safari' }, { 'browser_use/year': '2011', 'browser_use/browsers': 'firefox chrome' }] }] # yapf: disable mock_query_data.return_value = data csv_df_builder = CSVDataFrameBuilder(self.user.username, xform.id_string, split_select_multiples=False, include_images=False) # pylint: disable=protected-access cursor = [row for row in csv_df_builder._query_data()] result = [k for k in csv_df_builder._format_for_dataframe(cursor)] expected_result = [{ 'name': 'Tom', 'age': 23, 'browser_use[1]/year': '2010', 'browser_use[1]/browsers': 'firefox safari', 'browser_use[2]/year': '2011', 'browser_use[2]/browsers': 'firefox chrome' }] self.maxDiff = None self.assertEqual(expected_result, result)
def data(self, request, **kwargs): """ Streams submission data response matching uuid in the request. """ self.object = self.get_object() # get greater than value and cast it to an int gt_id = request.query_params.get('gt_id') gt_id = gt_id and parse_int(gt_id) count = request.query_params.get('count') pagination_keys = [ self.paginator.page_query_param, self.paginator.page_size_query_param ] query_param_keys = request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) data = [] if isinstance(self.object.content_object, XForm): if not self.object.active: return Response(status=status.HTTP_404_NOT_FOUND) xform = self.object.content_object if xform.is_merged_dataset: qs_kwargs = { 'xform_id__in': list(xform.mergedxform.xforms.values_list('pk', flat=True)) } else: qs_kwargs = {'xform_id': xform.pk} if gt_id: qs_kwargs.update({'id__gt': gt_id}) instances = Instance.objects.filter(**qs_kwargs).order_by('pk') if count: return Response({'count': instances.count()}) if should_paginate: instances = self.paginate_queryset(instances) csv_df_builder = CSVDataFrameBuilder(xform.user.username, xform.id_string, include_images=False) data = csv_df_builder._format_for_dataframe( DataInstanceSerializer(instances, many=True).data) return self._get_streaming_response(data) return Response(data)
def data(self, request, **kwargs): self.object = self.get_object() # get greater than value and cast it to an int gt = request.query_params.get('gt_id') gt = gt and parse_int(gt) count = request.query_params.get('count') pagination_keys = [ self.paginator.page_query_param, self.paginator.page_size_query_param ] query_param_keys = request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) data = [] if isinstance(self.object.content_object, XForm): if not self.object.active: return Response(status=status.HTTP_404_NOT_FOUND) xform = self.object.content_object qs_kwargs = {'xform_id': xform.pk} if gt: qs_kwargs.update({'id__gt': gt}) instances = Instance.objects.filter(**qs_kwargs).order_by('pk') length = self.total_count = instances.count() if count: return Response({'count': self.total_count}) if should_paginate: instances = self.paginate_queryset(instances) length = 1 + self.paginator.page.end_index( ) - self.paginator.page.start_index() csv_df_builder = CSVDataFrameBuilder(xform.user.username, xform.id_string, include_images=False) data = csv_df_builder._format_for_dataframe( DataInstanceSerializer(instances, many=True).data, key_replacement_obj={ 'pattern': r"(/|-|\[|\])", "replacer": r"_" }) return self._get_streaming_response(data, length) return Response(data)
def data(self, request, **kwargs): """ Streams submission data response matching uuid in the request. """ self.object = self.get_object() # get greater than value and cast it to an int gt_id = request.query_params.get('gt_id') gt_id = gt_id and parse_int(gt_id) count = request.query_params.get('count') pagination_keys = [ self.paginator.page_query_param, self.paginator.page_size_query_param ] query_param_keys = request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) data = [] if isinstance(self.object.content_object, XForm): if not self.object.active: return Response(status=status.HTTP_404_NOT_FOUND) xform = self.object.content_object if xform.is_merged_dataset: qs_kwargs = {'xform_id__in': list( xform.mergedxform.xforms.values_list('pk', flat=True))} else: qs_kwargs = {'xform_id': xform.pk} if gt_id: qs_kwargs.update({'id__gt': gt_id}) instances = Instance.objects.filter(**qs_kwargs).order_by('pk') if count: return Response({'count': instances.count()}) if should_paginate: instances = self.paginate_queryset(instances) csv_df_builder = CSVDataFrameBuilder( xform.user.username, xform.id_string, include_images=False) data = csv_df_builder._format_for_dataframe( DataInstanceSerializer(instances, many=True).data) return self._get_streaming_response(data) return Response(data)
def to_flat_csv_export(self, path, data, username, id_string, filter_query, start=None, end=None): # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder csv_builder = CSVDataFrameBuilder(username, id_string, filter_query, self.GROUP_DELIMITER, self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, start, end, self.TRUNCATE_GROUP_TITLE) csv_builder.export_to(path)
def test_windows_excel_compatible_csv_export(self): """ Test window excel compatible CSV export. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False, win_excel_utf8=True) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name) csv_reader = csv.reader(csv_file) header = csv_reader.next() self.assertEqual(len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS)) expected_header = [ '\xef\xbb\xbfname', '\xef\xbb\xbfage', '\xef\xbb\xbfhas_kids', '\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfgps', '\xef\xbb\xbf_gps_latitude', '\xef\xbb\xbf_gps_longitude', '\xef\xbb\xbf_gps_altitude', '\xef\xbb\xbf_gps_precision', '\xef\xbb\xbfweb_browsers/firefox', '\xef\xbb\xbfweb_browsers/chrome', '\xef\xbb\xbfweb_browsers/ie', '\xef\xbb\xbfweb_browsers/safari', '\xef\xbb\xbfinstanceID', '\xef\xbb\xbf_id', '\xef\xbb\xbf_uuid', '\xef\xbb\xbf_submission_time', '\xef\xbb\xbf_tags', '\xef\xbb\xbf_notes', '\xef\xbb\xbf_version', '\xef\xbb\xbf_duration', '\xef\xbb\xbf_submitted_by', '\xef\xbb\xbf_total_media', '\xef\xbb\xbf_media_count', '\xef\xbb\xbf_media_all_received' ] self.assertEqual(expected_header, header) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_export_remove_group_name(self): """ Test CSV export with remove_group_name option. """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_images=False) # pylint: disable=protected-access record_count = csv_df_builder._query_data(count=True) self.assertEqual(record_count, 7) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received' ] self.assertEqual(expected_header, header) rows = [] for row in csv_reader: rows.append(row) self.assertEqual(len(rows), 7) self.assertEqual(rows[4][5], NA_REP) # close and delete file csv_file.close() os.unlink(temp_file.name)
def test_csv_dataframe_export_to(self): self._publish_nested_repeats_form() self._submit_fixture_instance( "nested_repeats", "01", submission_time=self._submission_time) self._submit_fixture_instance( "nested_repeats", "02", submission_time=self._submission_time) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_fixture_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "fixtures", "nested_repeats", "nested_repeats.csv" ) temp_file.close() fixture, output = '', '' with open(csv_fixture_path) as f: fixture = f.read() with open(temp_file.name) as f: output = f.read() os.unlink(temp_file.name) self.assertEqual(fixture, output)
def test_csv_export_extra_columns(self): """ Test CSV export EXTRA_COLUMNS """ self._publish_single_level_repeat_form() # submit 7 instances for _ in range(4): self._submit_fixture_instance("new_repeats", "01") self._submit_fixture_instance("new_repeats", "02") for _ in range(2): self._submit_fixture_instance("new_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_labels=True) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) expected_header = [ 'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name', 'kids_age', 'gps', '_gps_latitude', '_gps_longitude', '_gps_altitude', '_gps_precision', 'web_browsers/firefox', 'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari', 'instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received', '_xform_id' ] self.assertEqual(expected_header, header) # close and delete file csv_file.close() os.unlink(temp_file.name) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, remove_group_name=True, include_labels=True) temp_file = NamedTemporaryFile(suffix=".csv", delete=False) csv_df_builder.export_to(temp_file.name) csv_file = open(temp_file.name, 'r') csv_reader = csv.reader(csv_file) header = next(csv_reader) self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns)) self.assertEqual(expected_header, header) csv_file.close() os.unlink(temp_file.name)
def test_split_select_multiples_within_repeats(self): self.maxDiff = None record = { 'name': 'Tom', 'age': 23, 'browser_use': [ { 'browser_use/year': '2010', 'browser_use/browsers': 'firefox safari' }, { 'browser_use/year': '2011', 'browser_use/browsers': 'firefox chrome' } ] } expected_result = { 'name': 'Tom', 'age': 23, 'browser_use': [ { 'browser_use/year': '2010', 'browser_use/browsers/firefox': True, 'browser_use/browsers/safari': True, 'browser_use/browsers/ie': False, 'browser_use/browsers/chrome': False }, { 'browser_use/year': '2011', 'browser_use/browsers/firefox': True, 'browser_use/browsers/safari': False, 'browser_use/browsers/ie': False, 'browser_use/browsers/chrome': True } ] } select_multiples = { 'browser_use/browsers': [ 'browser_use/browsers/firefox', 'browser_use/browsers/safari', 'browser_use/browsers/ie', 'browser_use/browsers/chrome']} result = CSVDataFrameBuilder._split_select_multiples(record, select_multiples) self.assertEqual(expected_result, result)
def test_split_select_multiples_within_repeats(self): """ Test select multiples choices are split within repeats in CSV exports. """ self.maxDiff = None record = { 'name': 'Tom', 'age': 23, 'browser_use': [{ 'browser_use/year': '2010', 'browser_use/browsers': 'firefox safari' }, { 'browser_use/year': '2011', 'browser_use/browsers': 'firefox chrome' }] } # yapf: disable expected_result = { 'name': 'Tom', 'age': 23, 'browser_use': [{ 'browser_use/year': '2010', 'browser_use/browsers/firefox': True, 'browser_use/browsers/safari': True, 'browser_use/browsers/ie': False, 'browser_use/browsers/chrome': False }, { 'browser_use/year': '2011', 'browser_use/browsers/firefox': True, 'browser_use/browsers/safari': False, 'browser_use/browsers/ie': False, 'browser_use/browsers/chrome': True }] } # yapf: disable select_multiples = { 'browser_use/browsers': [ 'browser_use/browsers/firefox', 'browser_use/browsers/safari', 'browser_use/browsers/ie', 'browser_use/browsers/chrome' ] } # pylint: disable=protected-access result = CSVDataFrameBuilder._split_select_multiples( record, select_multiples) self.assertEqual(expected_result, result)
def test_split_select_multiples_within_repeats(self): self.maxDiff = None record = { 'name': 'Tom', 'age': 23, 'browser_use': [{ 'browser_use/year': '2010', 'browser_use/browsers': 'firefox safari' }, { 'browser_use/year': '2011', 'browser_use/browsers': 'firefox chrome' }] } expected_result = { 'name': 'Tom', 'age': 23, 'browser_use': [{ 'browser_use/year': '2010', 'browser_use/browsers/firefox': True, 'browser_use/browsers/safari': True, 'browser_use/browsers/ie': False, 'browser_use/browsers/chrome': False }, { 'browser_use/year': '2011', 'browser_use/browsers/firefox': True, 'browser_use/browsers/safari': False, 'browser_use/browsers/ie': False, 'browser_use/browsers/chrome': True }] } select_multiples = { 'browser_use/browsers': [ 'browser_use/browsers/firefox', 'browser_use/browsers/safari', 'browser_use/browsers/ie', 'browser_use/browsers/chrome' ] } result = CSVDataFrameBuilder._split_select_multiples( record, select_multiples) self.assertEqual(expected_result, result)
def test_split_select_multiples(self): self._publish_nested_repeats_form() self._submit_fixture_instance("nested_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) cursor = [k for k in csv_df_builder._query_data()] record = cursor[0] select_multiples = \ CSVDataFrameBuilder._collect_select_multiples(self.xform) result = CSVDataFrameBuilder._split_select_multiples( record, select_multiples) expected_result = { u'web_browsers/ie': True, u'web_browsers/safari': True, u'web_browsers/firefox': False, u'web_browsers/chrome': False } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, binary_select_multiples=True) result = csv_df_builder._split_select_multiples( record, select_multiples) expected_result = { u'web_browsers/ie': 1, u'web_browsers/safari': 1, u'web_browsers/firefox': 0, u'web_browsers/chrome': 0 } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result)
def test_split_select_multiples(self): self._publish_nested_repeats_form() dd = self.xform.data_dictionary() self._submit_fixture_instance("nested_repeats", "01") csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) cursor = [k for k in csv_df_builder._query_data()] record = cursor[0] select_multiples = CSVDataFrameBuilder._collect_select_multiples(dd) result = CSVDataFrameBuilder._split_select_multiples(record, select_multiples) expected_result = { u'web_browsers/ie': True, u'web_browsers/safari': True, u'web_browsers/firefox': False, u'web_browsers/chrome': False } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result) csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, binary_select_multiples=True) result = csv_df_builder._split_select_multiples(record, select_multiples) expected_result = { u'web_browsers/ie': 1, u'web_browsers/safari': 1, u'web_browsers/firefox': 0, u'web_browsers/chrome': 0 } # build a new dictionary only composed of the keys we want to use in # the comparison result = dict([(key, result[key]) for key in result.keys() if key in expected_result.keys()]) self.assertEqual(expected_result, result)
def _csv_data_for_dataframe(self): csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False) cursor = csv_df_builder._query_data() return csv_df_builder._format_for_dataframe(cursor)
def _csv_data_for_dataframe(self): csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string) cursor = csv_df_builder._query_data() return csv_df_builder._format_for_dataframe(cursor)
def test_index_tag_replacement(self): """ Test that the default index tags are correctly replaced by an underscore """ self._publish_xls_fixture_set_xform("groups_in_repeats") self._submit_fixture_instance("groups_in_repeats", "01") self.xform.get_keys() csv_df_builder = CSVDataFrameBuilder(self.user.username, self.xform.id_string, include_images=False, index_tags=('_', '_')) cursor = csv_df_builder._query_data() result = [d for d in csv_df_builder._format_for_dataframe(cursor)][0] # remove dynamic fields ignore_list = [ '_uuid', 'meta/instanceID', 'formhub/uuid', '_submission_time', '_id', '_bamboo_dataset_id' ] for item in ignore_list: result.pop(item) expected_result = { u'_xform_id_string': u'groups_in_repeats', u'_xform_id': self.xform.pk, u'_status': u'submitted_via_web', u'_tags': u'', u'_notes': u'', u'_version': self.xform.version, u"_submitted_by": u'bob', u'name': u'Abe', u'age': 88, u'has_children': u'1', u'children_1_/childs_info/name': u'Cain', u'children_2_/childs_info/name': u'Abel', u'children_1_/childs_info/age': 56, u'children_2_/childs_info/age': 48, u'children_1_/immunization/immunization_received/polio_1': True, u'children_1_/immunization/immunization_received/polio_2': False, u'children_2_/immunization/immunization_received/polio_1': True, u'children_2_/immunization/immunization_received/polio_2': True, u'web_browsers/chrome': True, u'web_browsers/firefox': False, u'web_browsers/ie': False, u'web_browsers/safari': False, u'gps': u'-1.2626156 36.7923571 0.0 30.0', u'_geolocation': [-1.2626156, 36.7923571], u'_duration': '', u'_edited': False, u'_gps_latitude': u'-1.2626156', u'_gps_longitude': u'36.7923571', u'_gps_altitude': u'0.0', u'_gps_precision': u'30.0', u'_attachments': [], u'_total_media': 0, u'_media_count': 0, u'_media_all_received': True } self.maxDiff = None self.assertEqual(expected_result, result)