def test_parse_request_export_options(self):
request = self.factory.get('/export_async',
data={
"do_not_split_select_multiples":
"false",
"remove_group_name": "false",
"include_labels": "false",
"include_labels_only": "false",
"include_images": "false"
})
options = parse_request_export_options(request.GET)
self.assertEqual(options['split_select_multiples'], True)
self.assertEqual(options['include_labels'], False)
self.assertEqual(options['include_labels_only'], False)
self.assertEqual(options['remove_group_name'], False)
self.assertEqual(options['include_images'], False)
request = self.factory.get('/export_async',
data={
"do_not_split_select_multiples": "true",
"remove_group_name": "true",
"include_labels": "true",
"include_labels_only": "true",
"include_images": "true"
})
options = parse_request_export_options(request.GET)
self.assertEqual(options['split_select_multiples'], False)
self.assertEqual(options['include_labels'], True)
self.assertEqual(options['include_labels_only'], True)
self.assertEqual(options['remove_group_name'], True)
self.assertEqual(options['include_images'], True)
def export_async(self, request, *args, **kwargs):
job_uuid = request.query_params.get('job_uuid')
export_type = request.query_params.get('format')
query = request.query_params.get("query")
xform = self.get_object()
token = request.query_params.get('token')
meta = request.query_params.get('meta')
data_id = request.query_params.get('data_id')
options = parse_request_export_options(request.query_params)
options.update({
'meta': meta,
'token': token,
'data_id': data_id,
})
if query:
options.update({'query': query})
if request.query_params.get('format') in ['csvzip', 'savzip']:
# Overide renderer and mediatype because all response are
# suppose to be in json
# TODO: Avoid overiding the format query param for export type
# DRF uses format to select the renderer
self.request.accepted_renderer = renderers.JSONRenderer()
self.request.accepted_mediatype = 'application/json'
if job_uuid:
try:
resp = get_async_response(job_uuid, request, xform)
except Export.DoesNotExist:
# if this does not exist retry it against the primary
try:
with use_master:
resp = get_async_response(job_uuid, request, xform)
except NameError:
resp = get_async_response(job_uuid, request, xform)
else:
resp = process_async_export(request, xform, export_type, options)
if isinstance(resp, HttpResponseRedirect):
payload = {
"details": _("Google authorization needed"),
"url": resp.url
}
return Response(data=payload,
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
self.etag_data = '{}'.format(timezone.now())
return Response(data=resp,
status=status.HTTP_202_ACCEPTED,
content_type="application/json")
def test_parse_request_export_options(self):
request = self.factory.get(
'/export_async',
data={
"binary_select_multiples": "true",
"do_not_split_select_multiples": "false",
"remove_group_name": "false",
"include_labels": "false",
"include_labels_only": "false",
"include_images": "false"
})
options = parse_request_export_options(request.GET)
self.assertEqual(options['split_select_multiples'], True)
self.assertEqual(options['binary_select_multiples'], True)
self.assertEqual(options['include_labels'], False)
self.assertEqual(options['include_labels_only'], False)
self.assertEqual(options['remove_group_name'], False)
self.assertEqual(options['include_images'], False)
request = self.factory.get(
'/export_async',
data={
"do_not_split_select_multiples": "true",
"remove_group_name": "true",
"include_labels": "true",
"include_labels_only": "true",
"include_images": "true"
})
options = parse_request_export_options(request.GET)
self.assertEqual(options['split_select_multiples'], False)
self.assertEqual(options['include_labels'], True)
self.assertEqual(options['include_labels_only'], True)
self.assertEqual(options['remove_group_name'], True)
self.assertEqual(options['include_images'], True)
def export_async(self, request, *args, **kwargs):
job_uuid = request.query_params.get('job_uuid')
export_type = request.query_params.get('format')
query = request.query_params.get("query")
xform = self.get_object()
token = request.query_params.get('token')
meta = request.query_params.get('meta')
data_id = request.query_params.get('data_id')
options = parse_request_export_options(request.query_params)
options.update({
'meta': meta,
'token': token,
'data_id': data_id,
})
if query:
options.update({'query': query})
if job_uuid:
try:
resp = get_async_response(job_uuid, request, xform)
except Export.DoesNotExist:
# if this does not exist retry it against the primary
try:
with use_master:
resp = get_async_response(job_uuid, request, xform)
except NameError:
resp = get_async_response(job_uuid, request, xform)
else:
resp = process_async_export(request, xform, export_type, options)
if isinstance(resp, HttpResponseRedirect):
payload = {
"details": _("Google authorization needed"),
"url": resp.url
}
return Response(data=payload,
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
self.etag_data = '{}'.format(timezone.now())
return Response(data=resp,
status=status.HTTP_202_ACCEPTED,
content_type="application/json")
def export_async(self, request, *args, **kwargs):
job_uuid = request.query_params.get('job_uuid')
export_type = request.query_params.get('format')
query = request.query_params.get("query")
xform = self.get_object()
token = request.query_params.get('token')
meta = request.query_params.get('meta')
data_id = request.query_params.get('data_id')
options = parse_request_export_options(request.query_params)
options.update({
'meta': meta,
'token': token,
'data_id': data_id,
})
if query:
options.update({'query': query})
if job_uuid:
try:
resp = get_async_response(job_uuid, request, xform)
except Export.DoesNotExist:
# if this does not exist retry it against the primary
try:
with use_master:
resp = get_async_response(job_uuid, request, xform)
except NameError:
resp = get_async_response(job_uuid, request, xform)
else:
resp = process_async_export(request, xform, export_type, options)
if isinstance(resp, HttpResponseRedirect):
payload = {
"details": _("Google authorization needed"),
"url": resp.url
}
return Response(data=payload,
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
self.etag_data = '{}'.format(timezone.now())
return Response(data=resp,
status=status.HTTP_202_ACCEPTED,
content_type="application/json")
def custom_response_handler(request,
xform,
query,
export_type,
token=None,
meta=None,
dataview=False,
filename=None):
"""
Returns a HTTP response with export file for download.
"""
export_type = _get_export_type(export_type)
if export_type in EXTERNAL_EXPORT_TYPES and \
(token is not None) or (meta is not None):
export_type = Export.EXTERNAL_EXPORT
options = parse_request_export_options(request.query_params)
dataview_pk = hasattr(dataview, 'pk') and dataview.pk
options["dataview_pk"] = dataview_pk
if dataview:
columns_with_hxl = get_columns_with_hxl(xform.survey.get('children'))
if columns_with_hxl:
options['include_hxl'] = include_hxl_row(dataview.columns,
list(columns_with_hxl))
try:
query = filter_queryset_xform_meta_perms_sql(xform, request.user,
query)
except NoRecordsPermission:
return Response(data=json.dumps(
{"details": _("You don't have permission")}),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
if query:
options['query'] = query
remove_group_name = options.get("remove_group_name")
export_id = request.query_params.get("export_id")
if export_id:
export = get_object_or_404(Export, id=export_id, xform=xform)
else:
if export_type == Export.GOOGLE_SHEETS_EXPORT:
return Response(data=json.dumps(
{"details": _("Sheets export only supported in async mode")}),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
# check if we need to re-generate,
# we always re-generate if a filter is specified
def _new_export():
return _generate_new_export(request,
xform,
query,
export_type,
dataview_pk=dataview_pk)
if should_create_new_export(xform,
export_type,
options,
request=request):
export = _new_export()
else:
export = newest_export_for(xform, export_type, options)
if not export.filename and not export.error_message:
export = _new_export()
log_export(request, xform, export_type)
if export_type == Export.EXTERNAL_EXPORT:
return external_export_response(export)
if export.filename is None and export.error_message:
raise exceptions.ParseError(export.error_message)
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
_path, ext = os.path.splitext(export.filename)
ext = ext[1:]
show_date = True
if filename is None and export.status == Export.SUCCESSFUL:
filename = _generate_filename(request,
xform,
remove_group_name,
dataview_pk=dataview_pk)
else:
show_date = False
response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext],
filename,
extension=ext,
show_date=show_date,
file_path=export.filepath)
return response
def _generate_new_export(request,
xform,
query,
export_type,
dataview_pk=False):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
options = {
"extension": extension,
"username": xform.user.username,
"id_string": xform.id_string,
}
if query:
options['query'] = query
options["dataview_pk"] = dataview_pk
if export_type == Export.GOOGLE_SHEETS_EXPORT:
options['google_credentials'] = \
_get_google_credential(request).to_json()
try:
if export_type == Export.EXTERNAL_EXPORT:
options['token'] = request.GET.get('token')
options['data_id'] = request.GET.get('data_id')
options['meta'] = request.GET.get('meta')
export = generate_external_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.ZIP_EXPORT:
export = generate_attachments_zip_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
else:
options.update(parse_request_export_options(request.query_params))
export = generate_export(export_type, xform, None, options)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return async_status(FAILED, str(e))
except SPSSIOError as e:
raise exceptions.ParseError(str(e))
else:
return export
