def list(self, request, *args, **kwargs):
fields = request.GET.get("fields")
query = request.GET.get("query", {})
sort = request.GET.get("sort")
start = parse_int(request.GET.get("start"))
limit = parse_int(request.GET.get("limit"))
export_type = kwargs.get('format', request.GET.get("format"))
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
is_public_request = lookup == self.public_data_endpoint
if lookup_field not in kwargs.keys():
self.object_list = self.filter_queryset(self.get_queryset())
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
if is_public_request:
self.object_list = self._get_public_forms_queryset()
elif lookup:
qs = self.filter_queryset(self.get_queryset())
self.object_list = Instance.objects.filter(xform__in=qs,
deleted_at=None)
tags = self.request.QUERY_PARAMS.get('tags')
not_tagged = self.request.QUERY_PARAMS.get('not_tagged')
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
self.object_list = self.object_list.filter(tags__name__in=tags)
if not_tagged and isinstance(not_tagged, six.string_types):
not_tagged = not_tagged.split(',')
self.object_list = \
self.object_list.exclude(tags__name__in=not_tagged)
if (export_type is None or export_type in ['json']) \
and hasattr(self, 'object_list'):
return self._get_data(query, fields, sort, start, limit,
is_public_request)
xform = self.get_object()
if export_type == Attachment.OSM:
page = self.paginate_queryset(self.object_list)
serializer = self.get_pagination_serializer(page)
return Response(serializer.data)
elif export_type is None or export_type in ['json']:
# perform default viewset retrieve, no data export
return super(DataViewSet, self).list(request, *args, **kwargs)
elif export_type == 'geojson':
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
return custom_response_handler(request, xform, query, export_type)
def data(self, request, **kwargs):
"""
Streams submission data response matching uuid in the request.
"""
self.object = self.get_object()
# get greater than value and cast it to an int
gt_id = request.query_params.get('gt_id')
gt_id = gt_id and parse_int(gt_id)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
if xform.is_merged_dataset:
qs_kwargs = {
'xform_id__in':
list(xform.mergedxform.xforms.values_list('pk', flat=True))
}
else:
qs_kwargs = {'xform_id': xform.pk}
if gt_id:
qs_kwargs.update({'id__gt': gt_id})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
if count:
return Response({'count': instances.count()})
if should_paginate:
instances = self.paginate_queryset(instances)
csv_df_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string,
include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data)
return self._get_streaming_response(data)
return Response(data)
def get_serializer_class(self):
pk_lookup, dataid_lookup = self.lookup_fields
pk = self.kwargs.get(pk_lookup)
dataid = self.kwargs.get(dataid_lookup)
fmt = self.kwargs.get('format', self.request.GET.get("format"))
sort = self.request.GET.get("sort")
limit = parse_int(self.request.GET.get("limit"))
start = parse_int(self.request.GET.get("start"))
fields = self.request.GET.get("fields")
if fmt == Attachment.OSM:
serializer_class = OSMSerializer
elif fmt == 'geojson':
serializer_class = GeoJsonSerializer
elif pk is not None and dataid is None \
and pk != self.public_data_endpoint:
if sort or limit or start or fields:
serializer_class = JsonDataSerializer
else:
serializer_class = DataListSerializer
else:
serializer_class = \
super(DataViewSet, self).get_serializer_class()
return serializer_class
def data(self, request, **kwargs):
"""
Streams submission data response matching uuid in the request.
"""
self.object = self.get_object()
# get greater than value and cast it to an int
gt_id = request.query_params.get('gt_id')
gt_id = gt_id and parse_int(gt_id)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
if xform.is_merged_dataset:
qs_kwargs = {
'xform_id__in':
list(xform.mergedxform.xforms.values_list('pk', flat=True))
}
else:
qs_kwargs = {'xform_id': xform.pk}
if gt_id:
qs_kwargs.update({'id__gt': gt_id})
# Filter out deleted submissions
instances = Instance.objects.filter(
**qs_kwargs, deleted_at__isnull=True).order_by('pk')
if count:
return Response({'count': instances.count()})
if should_paginate:
instances = self.paginate_queryset(instances)
data = process_tableau_data(
TableauDataSerializer(instances, many=True).data, xform)
return self.get_streaming_response(data)
return Response(data)
def data(self, request, **kwargs):
self.object = self.get_object()
# get greater than value and cast it to an int
gt = request.query_params.get('gt_id')
gt = gt and parse_int(gt)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
qs_kwargs = {'xform_id': xform.pk}
if gt:
qs_kwargs.update({'id__gt': gt})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
length = self.total_count = instances.count()
if count:
return Response({'count': self.total_count})
if should_paginate:
instances = self.paginate_queryset(instances)
length = 1 + self.paginator.page.end_index(
) - self.paginator.page.start_index()
csv_df_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string,
include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data,
key_replacement_obj={
'pattern': r"(/|-|\[|\])",
"replacer": r"_"
})
return self._get_streaming_response(data, length)
return Response(data)
def data(self, request, **kwargs):
"""
Streams submission data response matching uuid in the request.
"""
self.object = self.get_object()
# get greater than value and cast it to an int
gt_id = request.query_params.get('gt_id')
gt_id = gt_id and parse_int(gt_id)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
if xform.is_merged_dataset:
qs_kwargs = {'xform_id__in': list(
xform.mergedxform.xforms.values_list('pk', flat=True))}
else:
qs_kwargs = {'xform_id': xform.pk}
if gt_id:
qs_kwargs.update({'id__gt': gt_id})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
if count:
return Response({'count': instances.count()})
if should_paginate:
instances = self.paginate_queryset(instances)
csv_df_builder = CSVDataFrameBuilder(
xform.user.username, xform.id_string, include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data)
return self._get_streaming_response(data)
return Response(data)
def list(self, request, *args, **kwargs):
fields = request.GET.get("fields")
query = request.GET.get("query", {})
sort = request.GET.get("sort")
start = parse_int(request.GET.get("start"))
limit = parse_int(request.GET.get("limit"))
export_type = kwargs.get('format', request.GET.get("format"))
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
is_public_request = lookup == self.public_data_endpoint
if lookup_field not in list(kwargs):
self.object_list = self.filter_queryset(self.get_queryset())
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
if is_public_request:
self.object_list = self._get_public_forms_queryset()
elif lookup:
qs = self.filter_queryset(self.get_queryset()).values_list(
'pk', 'is_merged_dataset')
xform_id, is_merged_dataset = qs[0] if qs else (lookup, False)
pks = [xform_id]
if is_merged_dataset:
merged_form = MergedXForm.objects.get(pk=xform_id)
qs = merged_form.xforms.filter(
deleted_at__isnull=True).values_list(
'id', 'num_of_submissions')
try:
pks, num_of_submissions = [
list(value) for value in zip(*qs)
]
num_of_submissions = sum(num_of_submissions)
except ValueError:
pks, num_of_submissions = [], 0
else:
num_of_submissions = XForm.objects.get(
id=xform_id).num_of_submissions
self.object_list = Instance.objects.filter(
xform_id__in=pks, deleted_at=None).only('json')
# Enable ordering for XForms with Submissions that are less
# than the SUBMISSION_RETRIEVAL_THRESHOLD
if num_of_submissions < SUBMISSION_RETRIEVAL_THRESHOLD:
self.object_list = self.object_list.order_by('id')
xform = self.get_object()
self.object_list = \
filter_queryset_xform_meta_perms(xform, request.user,
self.object_list)
tags = self.request.query_params.get('tags')
not_tagged = self.request.query_params.get('not_tagged')
self.object_list = filters.InstanceFilter(
self.request.query_params,
queryset=self.object_list,
request=request).qs
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
self.object_list = self.object_list.filter(tags__name__in=tags)
if not_tagged and isinstance(not_tagged, six.string_types):
not_tagged = not_tagged.split(',')
self.object_list = \
self.object_list.exclude(tags__name__in=not_tagged)
if (
export_type is None or
export_type in ['json', 'jsonp', 'debug', 'xml']) \
and hasattr(self, 'object_list'):
return self._get_data(query, fields, sort, start, limit,
is_public_request)
xform = self.get_object()
kwargs = {'instance__xform': xform}
if export_type == Attachment.OSM:
if request.GET:
self.set_object_list(query, fields, sort, start, limit,
is_public_request)
kwargs = {'instance__in': self.object_list}
osm_list = OsmData.objects.filter(**kwargs).order_by('instance')
page = self.paginate_queryset(osm_list)
serializer = self.get_serializer(page)
return Response(serializer.data)
elif export_type is None or export_type in ['json']:
# perform default viewset retrieve, no data export
return super(DataViewSet, self).list(request, *args, **kwargs)
elif export_type == 'geojson':
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
return custom_response_handler(request, xform, query, export_type)
def list(self, request, *args, **kwargs):
fields = request.GET.get("fields")
query = request.GET.get("query", {})
sort = request.GET.get("sort")
start = parse_int(request.GET.get("start"))
limit = parse_int(request.GET.get("limit"))
export_type = kwargs.get('format', request.GET.get("format"))
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
is_public_request = lookup == self.public_data_endpoint
if lookup_field not in kwargs.keys():
self.object_list = self.filter_queryset(self.get_queryset())
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
if is_public_request:
self.object_list = self._get_public_forms_queryset()
elif lookup:
qs = self.filter_queryset(self.get_queryset()).values_list(
'pk', flat=True)
xform_id = qs[0] if qs else lookup
self.object_list = Instance.objects.filter(
xform_id=xform_id, deleted_at=None).only('json')
xform = self.get_object()
self.object_list = \
filter_queryset_xform_meta_perms(xform, request.user,
self.object_list)
tags = self.request.query_params.get('tags')
not_tagged = self.request.query_params.get('not_tagged')
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
self.object_list = self.object_list.filter(tags__name__in=tags)
if not_tagged and isinstance(not_tagged, six.string_types):
not_tagged = not_tagged.split(',')
self.object_list = \
self.object_list.exclude(tags__name__in=not_tagged)
if (export_type is None or export_type in ['json', 'jsonp', 'debug']) \
and hasattr(self, 'object_list'):
return self._get_data(query, fields, sort, start, limit,
is_public_request)
xform = self.get_object()
kwargs = {'instance__xform': xform}
if export_type == Attachment.OSM:
if request.GET:
self.set_object_list_and_total_count(query, fields, sort,
start, limit,
is_public_request)
kwargs = {'instance__in': self.object_list}
osm_list = OsmData.objects.filter(**kwargs)
page = self.paginate_queryset(osm_list)
serializer = self.get_serializer(page)
return Response(serializer.data)
elif export_type is None or export_type in ['json']:
# perform default viewset retrieve, no data export
return super(DataViewSet, self).list(request, *args, **kwargs)
elif export_type == 'geojson':
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
return custom_response_handler(request, xform, query, export_type)
def list(self, request, *args, **kwargs):
fields = request.GET.get("fields")
query = request.GET.get("query", {})
sort = request.GET.get("sort")
start = parse_int(request.GET.get("start"))
limit = parse_int(request.GET.get("limit"))
export_type = kwargs.get('format', request.GET.get("format"))
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
is_public_request = lookup == self.public_data_endpoint
if lookup_field not in list(kwargs):
self.object_list = self.filter_queryset(self.get_queryset())
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
if is_public_request:
self.object_list = self._get_public_forms_queryset()
elif lookup:
qs = self.filter_queryset(
self.get_queryset()
).values_list('pk', 'is_merged_dataset')
xform_id, is_merged_dataset = qs[0] if qs else (lookup, False)
pks = [xform_id]
if is_merged_dataset:
xforms = MergedXForm.objects.filter(
pk=xform_id, xforms__deleted_at__isnull=True)\
.values_list('xforms', flat=True)
pks = [pk for pk in xforms if pk] or [xform_id]
self.object_list = Instance.objects.filter(
xform_id__in=pks, deleted_at=None).only('json').order_by('id')
xform = self.get_object()
self.object_list = \
filter_queryset_xform_meta_perms(xform, request.user,
self.object_list)
tags = self.request.query_params.get('tags')
not_tagged = self.request.query_params.get('not_tagged')
self.object_list = filters.InstanceFilter(
self.request.query_params,
queryset=self.object_list,
request=request
).qs
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
self.object_list = self.object_list.filter(tags__name__in=tags)
if not_tagged and isinstance(not_tagged, six.string_types):
not_tagged = not_tagged.split(',')
self.object_list = \
self.object_list.exclude(tags__name__in=not_tagged)
if (export_type is None or export_type in ['json', 'jsonp', 'debug']) \
and hasattr(self, 'object_list'):
return self._get_data(query, fields, sort, start, limit,
is_public_request)
xform = self.get_object()
kwargs = {'instance__xform': xform}
if export_type == Attachment.OSM:
if request.GET:
self.set_object_list(
query, fields, sort, start, limit, is_public_request)
kwargs = {'instance__in': self.object_list}
osm_list = OsmData.objects.filter(**kwargs).order_by('instance')
page = self.paginate_queryset(osm_list)
serializer = self.get_serializer(page)
return Response(serializer.data)
elif export_type is None or export_type in ['json']:
# perform default viewset retrieve, no data export
return super(DataViewSet, self).list(request, *args, **kwargs)
elif export_type == 'geojson':
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
return custom_response_handler(request, xform, query, export_type)
