def export_async(self, request, *args, **kwargs):
params = request.query_params
job_uuid = params.get('job_uuid')
export_type = params.get('format')
include_hxl = params.get('include_hxl', False)
include_labels = params.get('include_labels', False)
include_labels_only = params.get('include_labels_only', False)
query = params.get("query")
dataview = self.get_object()
xform = dataview.xform
if include_labels is not None:
include_labels = str_to_bool(include_labels)
if include_labels_only is not None:
include_labels_only = str_to_bool(include_labels_only)
if include_hxl is not None:
include_hxl = str_to_bool(include_hxl)
remove_group_name = params.get('remove_group_name', False)
columns_with_hxl = get_columns_with_hxl(xform.survey.get('children'))
if columns_with_hxl and include_hxl:
include_hxl = include_hxl_row(
dataview.columns, list(columns_with_hxl)
)
options = {
'remove_group_name': remove_group_name,
'dataview_pk': dataview.pk,
'include_hxl': include_hxl,
'include_labels': include_labels,
'include_labels_only': include_labels_only
}
if query:
options.update({'query': query})
if job_uuid:
job = AsyncResult(job_uuid)
if job.state == 'SUCCESS':
export_id = job.result
export = Export.objects.get(id=export_id)
resp = export_async_export_response(request, export)
else:
resp = {
'job_status': job.state
}
else:
resp = process_async_export(request, xform, export_type,
options=options)
return Response(data=resp,
status=status.HTTP_202_ACCEPTED,
content_type="application/json")
def data(self, request, format='json', **kwargs):
"""Retrieve the data from the xform using this dataview"""
start = request.GET.get("start")
limit = request.GET.get("limit")
count = request.GET.get("count")
sort = request.GET.get("sort")
query = request.GET.get("query")
export_type = self.kwargs.get('format', request.GET.get("format"))
self.object = self.get_object()
if export_type is None or export_type in ['json', 'debug']:
data = DataView.query_data(self.object,
start,
limit,
str_to_bool(count),
sort=sort,
filter_query=query)
if 'error' in data:
raise ParseError(data.get('error'))
serializer = self.get_serializer(data, many=True)
return Response(serializer.data)
else:
return custom_response_handler(request,
self.object.xform,
query,
export_type,
dataview=self.object)
def forms(self, request, **kwargs):
"""Add a form to a project or list forms for the project.
The request key `xls_file` holds the XLSForm file object.
"""
project = self.object = self.get_object()
if request.method.upper() == 'POST':
survey = utils.publish_project_xform(request, project)
if isinstance(survey, XForm):
if 'formid' in request.data:
serializer_cls = XFormSerializer
else:
serializer_cls = XFormCreateSerializer
serializer = serializer_cls(survey,
context={'request': request})
published_by_formbuilder = request.data.get(
'published_by_formbuilder'
)
if str_to_bool(published_by_formbuilder):
MetaData.published_by_formbuilder(survey, 'True')
return Response(serializer.data,
status=status.HTTP_201_CREATED)
return Response(survey, status=status.HTTP_400_BAD_REQUEST)
xforms = XForm.objects.filter(project=project)
serializer = XFormSerializer(xforms, context={'request': request},
many=True)
return Response(serializer.data)
def forms(self, request, **kwargs):
"""Add a form to a project or list forms for the project.
The request key `xls_file` holds the XLSForm file object.
"""
project = self.object = self.get_object()
if request.method.upper() == 'POST':
survey = utils.publish_project_xform(request, project)
if isinstance(survey, XForm):
if 'formid' in request.data:
serializer_cls = XFormSerializer
else:
serializer_cls = XFormCreateSerializer
serializer = serializer_cls(survey,
context={'request': request})
published_by_formbuilder = request.data.get(
'published_by_formbuilder'
)
if str_to_bool(published_by_formbuilder):
MetaData.published_by_formbuilder(survey, 'True')
return Response(serializer.data,
status=status.HTTP_201_CREATED)
return Response(survey, status=status.HTTP_400_BAD_REQUEST)
xforms = XForm.objects.filter(project=project)
serializer = XFormSerializer(xforms, context={'request': request},
many=True)
return Response(serializer.data)
def data(self, request, format='json', **kwargs):
"""Retrieve the data from the xform using this dataview"""
start = request.GET.get("start")
limit = request.GET.get("limit")
count = request.GET.get("count")
sort = request.GET.get("sort")
query = request.GET.get("query")
export_type = self.kwargs.get('format', request.GET.get("format"))
self.object = self.get_object()
if export_type is None or export_type in ['json', 'debug']:
data = DataView.query_data(self.object, start, limit,
str_to_bool(count), sort=sort,
filter_query=query)
if 'error' in data:
raise ParseError(data.get('error'))
serializer = self.get_serializer(data, many=True)
return Response(serializer.data)
else:
return custom_response_handler(request, self.object.xform, query,
export_type,
dataview=self.object)
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(export_type, xform.user.username,
xform.id_string, None,
request.GET.get('token'), query,
request.GET.get('meta'),
request.GET.get('data_id'))
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(export_type,
extension,
xform.user.username,
xform.id_string,
export_id=None,
filter_query=None)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(export_type,
extension,
xform.user.username,
xform.id_string,
export_id=None,
filter_query=None)
else:
remove_group_name = False
if "remove_group_name" in request.QUERY_PARAMS:
remove_group_name = \
str_to_bool(request.QUERY_PARAMS["remove_group_name"])
export = generate_export(export_type,
extension,
xform.user.username,
xform.id_string,
None,
query,
remove_group_name=remove_group_name)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(
export_type, xform.user.username,
xform.id_string, None, request.GET.get('token'), query,
request.GET.get('meta'), request.GET.get('data_id')
)
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(
export_type, extension, xform.user.username,
xform.id_string, export_id=None, filter_query=None)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(
export_type, extension, xform.user.username,
xform.id_string, export_id=None, filter_query=None)
else:
remove_group_name = False
if "remove_group_name" in request.QUERY_PARAMS:
remove_group_name = \
str_to_bool(request.QUERY_PARAMS["remove_group_name"])
export = generate_export(
export_type, extension, xform.user.username,
xform.id_string, None, query,
remove_group_name=remove_group_name
)
audit = {
"xform": xform.id_string,
"export_type": export_type
}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def create_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username__iexact=username)
xform = get_form({'user': owner, 'id_string__iexact': id_string})
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if export_type == Export.EXTERNAL_EXPORT:
# check for template before trying to generate a report
if not MetaData.external_export(xform):
return HttpResponseForbidden(_(u'No XLS Template set.'))
credential = None
if export_type == Export.GOOGLE_SHEETS_EXPORT:
credential = _get_google_credential(request)
if isinstance(credential, HttpResponseRedirect):
return credential
query = request.POST.get("query")
force_xlsx = request.POST.get('xls') != 'true'
# export options
group_delimiter = request.POST.get("options[group_delimiter]", '/')
if group_delimiter not in ['.', '/']:
return HttpResponseBadRequest(
_("%s is not a valid delimiter" % group_delimiter))
# default is True, so when dont_.. is yes
# split_select_multiples becomes False
split_select_multiples = request.POST.get(
"options[dont_split_select_multiples]", "no") == "no"
binary_select_multiples = getattr(settings, 'BINARY_SELECT_MULTIPLES',
False)
remove_group_name = request.POST.get("options[remove_group_name]", "false")
# external export option
meta = request.POST.get("meta")
options = {
'group_delimiter': group_delimiter,
'split_select_multiples': split_select_multiples,
'binary_select_multiples': binary_select_multiples,
'remove_group_name': str_to_bool(remove_group_name),
'meta': meta.replace(",", "") if meta else None,
'google_credentials': credential
}
try:
create_async_export(xform, export_type, query, force_xlsx, options)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
else:
audit = {"xform": xform.id_string, "export_type": export_type}
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'export_type': export_type.upper(),
'id_string': xform.id_string,
}, audit, request)
return HttpResponseRedirect(
reverse(export_list,
kwargs={
"username": username,
"id_string": id_string,
"export_type": export_type
}))
def test_str_to_bool(self):
self.assertTrue(str_to_bool(True))
self.assertTrue(str_to_bool('True'))
self.assertTrue(str_to_bool('TRUE'))
self.assertTrue(str_to_bool('true'))
self.assertTrue(str_to_bool('t'))
self.assertTrue(str_to_bool('1'))
self.assertTrue(str_to_bool(1))
self.assertFalse(str_to_bool(False))
self.assertFalse(str_to_bool('False'))
self.assertFalse(str_to_bool('F'))
self.assertFalse(str_to_bool('random'))
self.assertFalse(str_to_bool(234))
self.assertFalse(str_to_bool(0))
self.assertFalse(str_to_bool('0'))
def test_str_to_bool(self):
self.assertTrue(str_to_bool(True))
self.assertTrue(str_to_bool('True'))
self.assertTrue(str_to_bool('TRUE'))
self.assertTrue(str_to_bool('true'))
self.assertTrue(str_to_bool('t'))
self.assertTrue(str_to_bool('1'))
self.assertTrue(str_to_bool(1))
self.assertFalse(str_to_bool(False))
self.assertFalse(str_to_bool('False'))
self.assertFalse(str_to_bool('F'))
self.assertFalse(str_to_bool('random'))
self.assertFalse(str_to_bool(234))
self.assertFalse(str_to_bool(0))
self.assertFalse(str_to_bool('0'))
def create_export(request, username, id_string, export_type):
"""
Create async export tasks view.
"""
owner = get_object_or_404(User, username__iexact=username)
xform = get_form({'user': owner, 'id_string__iexact': id_string})
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if export_type == Export.EXTERNAL_EXPORT:
# check for template before trying to generate a report
if not MetaData.external_export(xform):
return HttpResponseForbidden(_(u'No XLS Template set.'))
credential = None
if export_type == Export.GOOGLE_SHEETS_EXPORT:
credential = _get_google_credential(request)
if isinstance(credential, HttpResponseRedirect):
return credential
query = request.POST.get("query")
force_xlsx = request.POST.get('xls') != 'true'
# export options
group_delimiter = request.POST.get("options[group_delimiter]", '/')
if group_delimiter not in ['.', '/']:
return HttpResponseBadRequest(
_("%s is not a valid delimiter" % group_delimiter))
# default is True, so when dont_.. is yes
# split_select_multiples becomes False
split_select_multiples = request.POST.get(
"options[dont_split_select_multiples]", "no") == "no"
binary_select_multiples = getattr(settings, 'BINARY_SELECT_MULTIPLES',
False)
remove_group_name = request.POST.get("options[remove_group_name]", "false")
value_select_multiples = request.POST.get(
"options[value_select_multiples]", "false")
# external export option
meta = request.POST.get("meta")
options = {
'group_delimiter': group_delimiter,
'split_select_multiples': split_select_multiples,
'binary_select_multiples': binary_select_multiples,
'value_select_multiples': str_to_bool(value_select_multiples),
'remove_group_name': str_to_bool(remove_group_name),
'meta': meta.replace(",", "") if meta else None,
'google_credentials': credential
}
try:
create_async_export(xform, export_type, query, force_xlsx, options)
except ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
else:
audit = {"xform": xform.id_string, "export_type": export_type}
audit_log(Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'export_type': export_type.upper(),
'id_string': xform.id_string,
}, audit, request)
return HttpResponseRedirect(
reverse(
export_list,
kwargs={
"username": username,
"id_string": id_string,
"export_type": export_type
}))
def create_async_export(xform, export_type, query, force_xlsx, options=None):
username = xform.user.username
id_string = xform.id_string
@transaction.commit_on_success
def _create_export(xform, export_type):
return Export.objects.create(xform=xform, export_type=export_type)
export = _create_export(xform, export_type)
result = None
arguments = {
'username': username,
'id_string': id_string,
'export_id': export.id,
'query': query,
}
if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT,
Export.CSV_EXPORT, Export.CSV_ZIP_EXPORT,
Export.SAV_ZIP_EXPORT]:
if options and "group_delimiter" in options:
arguments["group_delimiter"] = options["group_delimiter"]
if options and "split_select_multiples" in options:
arguments["split_select_multiples"] =\
options["split_select_multiples"]
if options and "binary_select_multiples" in options:
arguments["binary_select_multiples"] =\
options["binary_select_multiples"]
if options and "remove_group_name" in options:
arguments["remove_group_name"] = \
str_to_bool(options["remove_group_name"])
# start async export
if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT]:
result = create_xls_export.apply_async((), arguments)
elif export_type == Export.CSV_EXPORT:
result = create_csv_export.apply_async(
(), arguments)
elif export_type == Export.CSV_ZIP_EXPORT:
result = create_csv_zip_export.apply_async(
(), arguments)
elif export_type == Export.SAV_ZIP_EXPORT:
result = create_sav_zip_export.apply_async(
(), arguments)
else:
raise Export.ExportTypeError
elif export_type == Export.ZIP_EXPORT:
# start async export
result = create_zip_export.apply_async(
(), arguments)
elif export_type == Export.KML_EXPORT:
# start async export
result = create_kml_export.apply_async(
(), arguments)
elif export_type == Export.OSM_EXPORT:
# start async export
result = create_osm_export.apply_async(
(), arguments)
elif export_type == Export.EXTERNAL_EXPORT:
if options and "token" in options:
arguments["token"] = options["token"]
if options and "meta" in options:
arguments["meta"] = options["meta"]
if options and "data_id" in options:
arguments["data_id"] = options["data_id"]
result = create_external_export.apply_async(
(), arguments)
else:
raise Export.ExportTypeError
if result:
# when celery is running eager, the export has been generated by the
# time we get here so lets retrieve the export object a fresh before we
# save
if settings.CELERY_ALWAYS_EAGER:
export = Export.objects.get(id=export.id)
export.task_id = result.task_id
export.save()
return export, result
return None
