def retrieve(self, request, *args, **kwargs):
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
if lookup == self.public_forms_endpoint:
self.object_list = self._get_public_forms_queryset()
page = self.paginate_queryset(self.object_list)
if page is not None:
serializer = self.get_pagination_serializer(page)
else:
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
xform = self.get_object()
export_type = kwargs.get('format') or \
request.query_params.get('format')
query = request.query_params.get("query")
token = request.GET.get('token')
meta = request.GET.get('meta')
if export_type is None or export_type in ['json', 'debug']:
# perform default viewset retrieve, no data export
return super(XFormViewSet, self).retrieve(request, *args, **kwargs)
return custom_response_handler(request, xform, query, export_type,
token, meta)
def data(self, request, format='json', **kwargs):
"""Retrieve the data from the xform using this dataview"""
start = request.GET.get("start")
limit = request.GET.get("limit")
count = request.GET.get("count")
sort = request.GET.get("sort")
query = request.GET.get("query")
export_type = self.kwargs.get('format', request.GET.get("format"))
self.object = self.get_object()
if export_type is None or export_type in ['json', 'debug']:
data = DataView.query_data(self.object,
start,
limit,
str_to_bool(count),
sort=sort,
filter_query=query)
if 'error' in data:
raise ParseError(data.get('error'))
serializer = self.get_serializer(data, many=True)
return Response(serializer.data)
else:
return custom_response_handler(request,
self.object.xform,
query,
export_type,
dataview=self.object)
def get_media_file_response(metadata, request=None):
"""
Returns a HTTP response for media files.
HttpResponse 200 if it represents a file on disk.
HttpResponseRedirect 302 incase the metadata represents a url.
HttpResponseNotFound 404 if the metadata file cannot be found.
"""
def get_data_value_objects(value):
"""
Returns a tuple of a DataView or XForm and the name of the media file.
Looks for 'dataview 123 fruits.csv' or 'xform 345 fruits.csv'.
"""
model = None
if value.startswith('dataview'):
model = DataView
elif value.startswith('xform'):
model = XForm
if model:
parts = value.split()
if len(parts) > 1:
name = parts[2] if len(parts) > 2 else None
return (get_object_or_404(model, pk=parts[1]), name)
return (None, None)
if metadata.data_file:
file_path = metadata.data_file.name
filename, extension = os.path.splitext(file_path.split('/')[-1])
extension = extension.strip('.')
dfs = get_storage_class()()
if dfs.exists(file_path):
response = response_with_mimetype_and_name(metadata.data_file_type,
filename,
extension=extension,
show_date=False,
file_path=file_path,
full_mime=True)
return response
return HttpResponseNotFound()
try:
URLValidator()(metadata.data_value)
except ValidationError:
obj, filename = get_data_value_objects(metadata.data_value)
if obj:
dataview = obj if isinstance(obj, DataView) else False
xform = obj.xform if isinstance(obj, DataView) else obj
return custom_response_handler(request,
xform, {},
Export.CSV_EXPORT,
filename=filename,
dataview=dataview)
return HttpResponseRedirect(metadata.data_value)
def retrieve(self, request, *args, **kwargs):
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
if lookup == self.public_forms_endpoint:
self.object_list = self._get_public_forms_queryset()
page = self.paginate_queryset(self.object_list)
if page is not None:
serializer = self.get_pagination_serializer(page)
else:
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
xform = self.get_object()
export_type = kwargs.get('format') or \
request.query_params.get('format')
query = request.query_params.get("query")
token = request.GET.get('token')
meta = request.GET.get('meta')
if export_type is None or export_type in ['json', 'debug']:
# perform default viewset retrieve, no data export
return super(XFormViewSet, self).retrieve(request, *args, **kwargs)
return custom_response_handler(request,
xform,
query,
export_type,
token,
meta)
def data(self, request, format='json', **kwargs):
"""Retrieve the data from the xform using this dataview"""
start = request.GET.get("start")
limit = request.GET.get("limit")
count = request.GET.get("count")
sort = request.GET.get("sort")
query = request.GET.get("query")
export_type = self.kwargs.get('format', request.GET.get("format"))
self.object = self.get_object()
if export_type is None or export_type in ['json', 'debug']:
data = DataView.query_data(self.object, start, limit,
str_to_bool(count), sort=sort,
filter_query=query)
if 'error' in data:
raise ParseError(data.get('error'))
serializer = self.get_serializer(data, many=True)
return Response(serializer.data)
else:
return custom_response_handler(request, self.object.xform, query,
export_type,
dataview=self.object)
def xls_export(self, request, *args, **kwargs):
dataview = self.get_object()
xform = dataview.xform
token = None
export_type = "xls"
query = request.query_params.get("query", {})
meta = request.GET.get('meta')
return custom_response_handler(request, xform, query, export_type,
token, meta, dataview)
def get_media_file_response(metadata, request=None):
def get_data_value_objects(value):
model = None
if value.startswith('dataview'):
model = DataView
elif value.startswith('xform'):
model = XForm
if model:
parts = value.split()
if len(parts) > 1:
name = parts[2] if len(parts) > 2 else None
return (get_object_or_404(model, pk=parts[1]), name)
return (None, None)
if metadata.data_file:
file_path = metadata.data_file.name
filename, extension = os.path.splitext(file_path.split('/')[-1])
extension = extension.strip('.')
dfs = get_storage_class()()
if dfs.exists(file_path):
response = response_with_mimetype_and_name(metadata.data_file_type,
filename,
extension=extension,
show_date=False,
file_path=file_path,
full_mime=True)
return response
else:
return HttpResponseNotFound()
else:
try:
URLValidator()(metadata.data_value)
except ValidationError:
obj, filename = get_data_value_objects(metadata.data_value)
if obj:
dataview = obj if isinstance(obj, DataView) else False
xform = obj.xform if isinstance(obj, DataView) else obj
return custom_response_handler(request,
xform, {},
Export.CSV_EXPORT,
filename=filename,
dataview=dataview)
return HttpResponseRedirect(metadata.data_value)
def xls_export(self, request, *args, **kwargs):
dataview = self.get_object()
xform = dataview.xform
token = None
export_type = "xls"
query = request.query_params.get("query", {})
meta = request.GET.get('meta')
return custom_response_handler(request,
xform,
query,
export_type,
token,
meta,
dataview)
def list(self, request, *args, **kwargs):
fields = request.GET.get("fields")
query = request.GET.get("query", {})
sort = request.GET.get("sort")
start = parse_int(request.GET.get("start"))
limit = parse_int(request.GET.get("limit"))
export_type = kwargs.get('format', request.GET.get("format"))
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
is_public_request = lookup == self.public_data_endpoint
if lookup_field not in list(kwargs):
self.object_list = self.filter_queryset(self.get_queryset())
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
if is_public_request:
self.object_list = self._get_public_forms_queryset()
elif lookup:
qs = self.filter_queryset(self.get_queryset()).values_list(
'pk', 'is_merged_dataset')
xform_id, is_merged_dataset = qs[0] if qs else (lookup, False)
pks = [xform_id]
if is_merged_dataset:
merged_form = MergedXForm.objects.get(pk=xform_id)
qs = merged_form.xforms.filter(
deleted_at__isnull=True).values_list(
'id', 'num_of_submissions')
try:
pks, num_of_submissions = [
list(value) for value in zip(*qs)
]
num_of_submissions = sum(num_of_submissions)
except ValueError:
pks, num_of_submissions = [], 0
else:
num_of_submissions = XForm.objects.get(
id=xform_id).num_of_submissions
self.object_list = Instance.objects.filter(
xform_id__in=pks, deleted_at=None).only('json')
# Enable ordering for XForms with Submissions that are less
# than the SUBMISSION_RETRIEVAL_THRESHOLD
if num_of_submissions < SUBMISSION_RETRIEVAL_THRESHOLD:
self.object_list = self.object_list.order_by('id')
xform = self.get_object()
self.object_list = \
filter_queryset_xform_meta_perms(xform, request.user,
self.object_list)
tags = self.request.query_params.get('tags')
not_tagged = self.request.query_params.get('not_tagged')
self.object_list = filters.InstanceFilter(
self.request.query_params,
queryset=self.object_list,
request=request).qs
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
self.object_list = self.object_list.filter(tags__name__in=tags)
if not_tagged and isinstance(not_tagged, six.string_types):
not_tagged = not_tagged.split(',')
self.object_list = \
self.object_list.exclude(tags__name__in=not_tagged)
if (
export_type is None or
export_type in ['json', 'jsonp', 'debug', 'xml']) \
and hasattr(self, 'object_list'):
return self._get_data(query, fields, sort, start, limit,
is_public_request)
xform = self.get_object()
kwargs = {'instance__xform': xform}
if export_type == Attachment.OSM:
if request.GET:
self.set_object_list(query, fields, sort, start, limit,
is_public_request)
kwargs = {'instance__in': self.object_list}
osm_list = OsmData.objects.filter(**kwargs).order_by('instance')
page = self.paginate_queryset(osm_list)
serializer = self.get_serializer(page)
return Response(serializer.data)
elif export_type is None or export_type in ['json']:
# perform default viewset retrieve, no data export
return super(DataViewSet, self).list(request, *args, **kwargs)
elif export_type == 'geojson':
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
return custom_response_handler(request, xform, query, export_type)
def list(self, request, *args, **kwargs):
fields = request.GET.get("fields")
query = request.GET.get("query", {})
sort = request.GET.get("sort")
start = parse_int(request.GET.get("start"))
limit = parse_int(request.GET.get("limit"))
export_type = kwargs.get('format', request.GET.get("format"))
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
is_public_request = lookup == self.public_data_endpoint
if lookup_field not in kwargs.keys():
self.object_list = self.filter_queryset(self.get_queryset())
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
if is_public_request:
self.object_list = self._get_public_forms_queryset()
elif lookup:
qs = self.filter_queryset(self.get_queryset()).values_list(
'pk', flat=True)
xform_id = qs[0] if qs else lookup
self.object_list = Instance.objects.filter(
xform_id=xform_id, deleted_at=None).only('json')
xform = self.get_object()
self.object_list = \
filter_queryset_xform_meta_perms(xform, request.user,
self.object_list)
tags = self.request.query_params.get('tags')
not_tagged = self.request.query_params.get('not_tagged')
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
self.object_list = self.object_list.filter(tags__name__in=tags)
if not_tagged and isinstance(not_tagged, six.string_types):
not_tagged = not_tagged.split(',')
self.object_list = \
self.object_list.exclude(tags__name__in=not_tagged)
if (export_type is None or export_type in ['json', 'jsonp', 'debug']) \
and hasattr(self, 'object_list'):
return self._get_data(query, fields, sort, start, limit,
is_public_request)
xform = self.get_object()
kwargs = {'instance__xform': xform}
if export_type == Attachment.OSM:
if request.GET:
self.set_object_list_and_total_count(query, fields, sort,
start, limit,
is_public_request)
kwargs = {'instance__in': self.object_list}
osm_list = OsmData.objects.filter(**kwargs)
page = self.paginate_queryset(osm_list)
serializer = self.get_serializer(page)
return Response(serializer.data)
elif export_type is None or export_type in ['json']:
# perform default viewset retrieve, no data export
return super(DataViewSet, self).list(request, *args, **kwargs)
elif export_type == 'geojson':
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
return custom_response_handler(request, xform, query, export_type)
def list(self, request, *args, **kwargs):
fields = request.GET.get("fields")
query = request.GET.get("query", {})
sort = request.GET.get("sort")
start = parse_int(request.GET.get("start"))
limit = parse_int(request.GET.get("limit"))
export_type = kwargs.get('format', request.GET.get("format"))
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
is_public_request = lookup == self.public_data_endpoint
if lookup_field not in list(kwargs):
self.object_list = self.filter_queryset(self.get_queryset())
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
if is_public_request:
self.object_list = self._get_public_forms_queryset()
elif lookup:
qs = self.filter_queryset(
self.get_queryset()
).values_list('pk', 'is_merged_dataset')
xform_id, is_merged_dataset = qs[0] if qs else (lookup, False)
pks = [xform_id]
if is_merged_dataset:
xforms = MergedXForm.objects.filter(
pk=xform_id, xforms__deleted_at__isnull=True)\
.values_list('xforms', flat=True)
pks = [pk for pk in xforms if pk] or [xform_id]
self.object_list = Instance.objects.filter(
xform_id__in=pks, deleted_at=None).only('json').order_by('id')
xform = self.get_object()
self.object_list = \
filter_queryset_xform_meta_perms(xform, request.user,
self.object_list)
tags = self.request.query_params.get('tags')
not_tagged = self.request.query_params.get('not_tagged')
self.object_list = filters.InstanceFilter(
self.request.query_params,
queryset=self.object_list,
request=request
).qs
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
self.object_list = self.object_list.filter(tags__name__in=tags)
if not_tagged and isinstance(not_tagged, six.string_types):
not_tagged = not_tagged.split(',')
self.object_list = \
self.object_list.exclude(tags__name__in=not_tagged)
if (export_type is None or export_type in ['json', 'jsonp', 'debug']) \
and hasattr(self, 'object_list'):
return self._get_data(query, fields, sort, start, limit,
is_public_request)
xform = self.get_object()
kwargs = {'instance__xform': xform}
if export_type == Attachment.OSM:
if request.GET:
self.set_object_list(
query, fields, sort, start, limit, is_public_request)
kwargs = {'instance__in': self.object_list}
osm_list = OsmData.objects.filter(**kwargs).order_by('instance')
page = self.paginate_queryset(osm_list)
serializer = self.get_serializer(page)
return Response(serializer.data)
elif export_type is None or export_type in ['json']:
# perform default viewset retrieve, no data export
return super(DataViewSet, self).list(request, *args, **kwargs)
elif export_type == 'geojson':
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
return custom_response_handler(request, xform, query, export_type)
def get_media_file_response(metadata, request=None):
"""
Returns a HTTP response for media files.
HttpResponse 200 if it represents a file on disk.
HttpResponseRedirect 302 incase the metadata represents a url.
HttpResponseNotFound 404 if the metadata file cannot be found.
"""
def get_data_value_objects(value):
"""
Returns a tuple of a DataView or XForm and the name of the media file.
Looks for 'dataview 123 fruits.csv' or 'xform 345 fruits.csv'.
"""
model = None
if value.startswith('dataview'):
model = DataView
elif value.startswith('xform'):
model = XForm
if model:
parts = value.split()
if len(parts) > 1:
name = parts[2] if len(parts) > 2 else None
return (get_object_or_404(model, pk=parts[1]), name)
return (None, None)
if metadata.data_file:
file_path = metadata.data_file.name
filename, extension = os.path.splitext(file_path.split('/')[-1])
extension = extension.strip('.')
dfs = get_storage_class()()
if dfs.exists(file_path):
response = response_with_mimetype_and_name(
metadata.data_file_type,
filename,
extension=extension,
show_date=False,
file_path=file_path,
full_mime=True)
return response
return HttpResponseNotFound()
try:
URLValidator()(metadata.data_value)
except ValidationError:
obj, filename = get_data_value_objects(metadata.data_value)
if obj:
dataview = obj if isinstance(obj, DataView) else False
xform = obj.xform if isinstance(obj, DataView) else obj
return custom_response_handler(
request,
xform, {},
Export.CSV_EXPORT,
filename=filename,
dataview=dataview)
return HttpResponseRedirect(metadata.data_value)