def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
cursor = csv_df_builder._query_data()
return [d for d in csv_df_builder._format_for_dataframe(cursor)]
def test_no_split_select_multiples(self, mock_query_data):
"""
Test selct multiples are not split within repeats.
"""
md_xform = """
| survey |
| | type | name | label |
| | text | name | Name |
| | integer | age | Age |
| | begin repeat | browser_use | Browser Use |
| | integer | year | Year |
| | select_multiple browsers | browsers | Browsers |
| | end repeat | | |
| choices |
| | list name | name | label |
| | browsers | firefox | Firefox |
| | browsers | chrome | Chrome |
| | browsers | ie | Internet Explorer |
| | browsers | safari | Safari |
"""
xform = self._publish_markdown(md_xform, self.user, id_string='b')
data = [{
'name': 'Tom',
'age': 23,
'browser_use': [{
'browser_use/year': '2010',
'browser_use/browsers': 'firefox safari'
}, {
'browser_use/year': '2011',
'browser_use/browsers': 'firefox chrome'
}]
}] # yapf: disable
mock_query_data.return_value = data
csv_df_builder = CSVDataFrameBuilder(self.user.username,
xform.id_string,
split_select_multiples=False,
include_images=False)
# pylint: disable=protected-access
cursor = [row for row in csv_df_builder._query_data()]
result = [k for k in csv_df_builder._format_for_dataframe(cursor)]
expected_result = [{
'name': 'Tom',
'age': 23,
'browser_use[1]/year': '2010',
'browser_use[1]/browsers': 'firefox safari',
'browser_use[2]/year': '2011',
'browser_use[2]/browsers': 'firefox chrome'
}]
self.maxDiff = None
self.assertEqual(expected_result, result)
def data(self, request, **kwargs):
"""
Streams submission data response matching uuid in the request.
"""
self.object = self.get_object()
# get greater than value and cast it to an int
gt_id = request.query_params.get('gt_id')
gt_id = gt_id and parse_int(gt_id)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
if xform.is_merged_dataset:
qs_kwargs = {
'xform_id__in':
list(xform.mergedxform.xforms.values_list('pk', flat=True))
}
else:
qs_kwargs = {'xform_id': xform.pk}
if gt_id:
qs_kwargs.update({'id__gt': gt_id})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
if count:
return Response({'count': instances.count()})
if should_paginate:
instances = self.paginate_queryset(instances)
csv_df_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string,
include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data)
return self._get_streaming_response(data)
return Response(data)
def data(self, request, **kwargs):
self.object = self.get_object()
# get greater than value and cast it to an int
gt = request.query_params.get('gt_id')
gt = gt and parse_int(gt)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
qs_kwargs = {'xform_id': xform.pk}
if gt:
qs_kwargs.update({'id__gt': gt})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
length = self.total_count = instances.count()
if count:
return Response({'count': self.total_count})
if should_paginate:
instances = self.paginate_queryset(instances)
length = 1 + self.paginator.page.end_index(
) - self.paginator.page.start_index()
csv_df_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string,
include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data,
key_replacement_obj={
'pattern': r"(/|-|\[|\])",
"replacer": r"_"
})
return self._get_streaming_response(data, length)
return Response(data)
def data(self, request, **kwargs):
"""
Streams submission data response matching uuid in the request.
"""
self.object = self.get_object()
# get greater than value and cast it to an int
gt_id = request.query_params.get('gt_id')
gt_id = gt_id and parse_int(gt_id)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
if xform.is_merged_dataset:
qs_kwargs = {'xform_id__in': list(
xform.mergedxform.xforms.values_list('pk', flat=True))}
else:
qs_kwargs = {'xform_id': xform.pk}
if gt_id:
qs_kwargs.update({'id__gt': gt_id})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
if count:
return Response({'count': instances.count()})
if should_paginate:
instances = self.paginate_queryset(instances)
csv_df_builder = CSVDataFrameBuilder(
xform.user.username, xform.id_string, include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data)
return self._get_streaming_response(data)
return Response(data)
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
cursor = csv_df_builder._query_data()
return csv_df_builder._format_for_dataframe(cursor)
def test_index_tag_replacement(self):
"""
Test that the default index tags are correctly replaced by an
underscore
"""
self._publish_xls_fixture_set_xform("groups_in_repeats")
self._submit_fixture_instance("groups_in_repeats", "01")
self.xform.get_keys()
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False,
index_tags=('_', '_'))
cursor = csv_df_builder._query_data()
result = [d for d in csv_df_builder._format_for_dataframe(cursor)][0]
# remove dynamic fields
ignore_list = [
'_uuid', 'meta/instanceID', 'formhub/uuid', '_submission_time',
'_id', '_bamboo_dataset_id'
]
for item in ignore_list:
result.pop(item)
expected_result = {
u'_xform_id_string': u'groups_in_repeats',
u'_xform_id': self.xform.pk,
u'_status': u'submitted_via_web',
u'_tags': u'',
u'_notes': u'',
u'_version': self.xform.version,
u"_submitted_by": u'bob',
u'name': u'Abe',
u'age': 88,
u'has_children': u'1',
u'children_1_/childs_info/name': u'Cain',
u'children_2_/childs_info/name': u'Abel',
u'children_1_/childs_info/age': 56,
u'children_2_/childs_info/age': 48,
u'children_1_/immunization/immunization_received/polio_1': True,
u'children_1_/immunization/immunization_received/polio_2': False,
u'children_2_/immunization/immunization_received/polio_1': True,
u'children_2_/immunization/immunization_received/polio_2': True,
u'web_browsers/chrome': True,
u'web_browsers/firefox': False,
u'web_browsers/ie': False,
u'web_browsers/safari': False,
u'gps': u'-1.2626156 36.7923571 0.0 30.0',
u'_geolocation': [-1.2626156, 36.7923571],
u'_duration': '',
u'_edited': False,
u'_gps_latitude': u'-1.2626156',
u'_gps_longitude': u'36.7923571',
u'_gps_altitude': u'0.0',
u'_gps_precision': u'30.0',
u'_attachments': [],
u'_total_media': 0,
u'_media_count': 0,
u'_media_all_received': True
}
self.maxDiff = None
self.assertEqual(expected_result, result)
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
cursor = csv_df_builder._query_data()
return csv_df_builder._format_for_dataframe(cursor)
