def export_list(request, username, id_string, export_type):
if export_type == Export.GDOC_EXPORT:
redirect_url = reverse(
export_list,
kwargs={
'username': username, 'id_string': id_string,
'export_type': export_type})
token = _get_google_token(request, redirect_url)
if isinstance(token, HttpResponse):
return token
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__iexact=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if should_create_new_export(xform, export_type):
try:
create_async_export(
xform, export_type, query=None, force_xlsx=True)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
data = {
'username': owner.username,
'xform': xform,
'export_type': export_type,
'export_type_name': Export.EXPORT_TYPE_DICT[export_type],
'exports': Export.objects.filter(
xform=xform, export_type=export_type).order_by('-created_on')
}
return render(request, 'export_list.html', data)
def export_list(request, username, id_string, export_type):
if export_type == Export.GDOC_EXPORT:
redirect_url = reverse(
export_list,
kwargs={
'username': username, 'id_string': id_string,
'export_type': export_type})
token = _get_google_token(request, redirect_url)
if isinstance(token, HttpResponse):
return token
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if should_create_new_export(xform, export_type):
try:
create_async_export(
xform, export_type, query=None, force_xlsx=True)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
context = RequestContext(request)
context.username = owner.username
context.xform = xform
# TODO: better output e.g. Excel instead of XLS
context.export_type = export_type
context.export_type_name = Export.EXPORT_TYPE_DICT[export_type]
exports = Export.objects.filter(xform=xform, export_type=export_type)\
.order_by('-created_on')
context.exports = exports
return render_to_response('export_list.html', context_instance=context)
def export_list(request, username, id_string, export_type):
if export_type == Export.GDOC_EXPORT:
redirect_url = reverse(
export_list,
kwargs={
'username': username, 'id_string': id_string,
'export_type': export_type})
token = _get_google_token(request, redirect_url)
if isinstance(token, HttpResponse):
return token
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if should_create_new_export(xform, export_type):
try:
create_async_export(
xform, export_type, query=None, force_xlsx=True)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
context = RequestContext(request)
context.username = owner.username
context.xform = xform
# TODO: better output e.g. Excel instead of XLS
context.export_type = export_type
context.export_type_name = Export.EXPORT_TYPE_DICT[export_type]
exports = Export.objects.filter(xform=xform, export_type=export_type)\
.order_by('-created_on')
context.exports = exports
return render_to_response('export_list.html', context_instance=context)
def export_list(request, username, id_string, export_type):
if export_type == Export.GDOC_EXPORT:
redirect_url = reverse(
export_list,
kwargs={
'username': username, 'id_string': id_string,
'export_type': export_type})
token = _get_google_token(request, redirect_url)
if isinstance(token, HttpResponse):
return token
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if export_type == Export.EXTERNAL_EXPORT:
# check for template before trying to generate a report
if not MetaData.external_export(xform=xform):
return HttpResponseForbidden(_(u'No XLS Template set.'))
# Get meta and token
export_token = request.GET.get('token')
export_meta = request.GET.get('meta')
options = {
'meta': export_meta,
'token': export_token,
}
if should_create_new_export(xform, export_type):
try:
create_async_export(
xform, export_type, query=None, force_xlsx=True,
options=options)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
metadata = MetaData.objects.filter(xform=xform,
data_type="external_export")\
.values('id', 'data_value')
for m in metadata:
m['data_value'] = m.get('data_value').split('|')[0]
data = {
'username': owner.username,
'xform': xform,
'export_type': export_type,
'export_type_name': Export.EXPORT_TYPE_DICT[export_type],
'exports': Export.objects.filter(
xform=xform, export_type=export_type).order_by('-created_on'),
'metas': metadata
}
return render(request, 'export_list.html', data)
def test_should_create_new_export(self):
# should only create new export if filter is defined
# Test setup
export_type = "csv"
options = {"group_delimiter": "."}
self._publish_transportation_form_and_submit_instance()
will_create_new_export = should_create_new_export(
self.xform, export_type, options)
self.assertTrue(will_create_new_export)
def export_list(request, username, id_string, export_type):
if export_type == Export.GDOC_EXPORT:
redirect_url = reverse(
export_list,
kwargs={
'username': username, 'id_string': id_string,
'export_type': export_type})
token = _get_google_token(request, redirect_url)
if isinstance(token, HttpResponse):
return token
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if export_type == Export.EXTERNAL_EXPORT:
# check for template before trying to generate a report
if not MetaData.external_export(xform=xform):
return HttpResponseForbidden(_(u'No XLS Template set.'))
# Get meta and token
export_token = request.GET.get('token')
export_meta = request.GET.get('meta')
options = {
'meta': export_meta,
'token': export_token,
}
if should_create_new_export(xform, export_type):
try:
create_async_export(
xform, export_type, query=None, force_xlsx=True,
options=options)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
metadata = MetaData.objects.filter(xform=xform,
data_type="external_export")\
.values('id', 'data_value')
for m in metadata:
m['data_value'] = m.get('data_value').split('|')[0]
data = {
'username': owner.username,
'xform': xform,
'export_type': export_type,
'export_type_name': Export.EXPORT_TYPE_DICT[export_type],
'exports': Export.objects.filter(
xform=xform, export_type=export_type).order_by('-created_on'),
'metas': metadata
}
return render(request, 'export_list.html', data)
def test_should_create_new_export(self):
# should only create new export if filter is defined
# Test setup
export_type = "csv"
options = {"group_delimiter": "."}
self._publish_transportation_form_and_submit_instance()
will_create_new_export = should_create_new_export(
self.xform, export_type, options)
self.assertTrue(will_create_new_export)
def test_should_not_create_new_export_when_old_exists(self):
export_type = "csv"
self._publish_transportation_form_and_submit_instance()
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._create_old_export(self.xform, export_type, options)
will_create_new_export = should_create_new_export(
self.xform, export_type, options)
self.assertFalse(will_create_new_export)
def test_should_not_create_new_export_when_old_exists(self):
export_type = "csv"
self._publish_transportation_form_and_submit_instance()
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._create_old_export(self.xform, export_type, options)
will_create_new_export = should_create_new_export(
self.xform, export_type, options)
self.assertFalse(will_create_new_export)
def test_should_create_new_export_when_filter_defined(self):
export_type = "csv"
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._publish_transportation_form_and_submit_instance()
self._create_old_export(self.xform, export_type, options)
# Call should_create_new_export with updated options
options['remove_group_name'] = True
will_create_new_export = should_create_new_export(
self.xform, export_type, options)
self.assertTrue(will_create_new_export)
def test_should_create_new_export_when_filter_defined(self):
export_type = "csv"
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._publish_transportation_form_and_submit_instance()
self._create_old_export(self.xform, export_type, options)
# Call should_create_new_export with updated options
options['remove_group_name'] = True
will_create_new_export = should_create_new_export(
self.xform, export_type, options)
self.assertTrue(will_create_new_export)
def process_async_export(request, xform, export_type, options=None):
"""
Check if should generate export or just return the latest export.
Rules for regenerating an export are:
1. Filter included on the exports.
2. New submission done.
3. Always regenerate external exports.
(External exports uses templates and the template might have
changed)
:param request:
:param xform:
:param export_type:
:param options: additional export params that may include
query: export filter
token: template url for xls external reports
meta: metadataid that contains the external xls report template url
remove_group_name: Flag to determine if group names should appear
:return: response dictionary
"""
# maintain the order of keys while processing the export
export_type = _get_export_type(export_type)
token = options.get("token")
meta = options.get("meta")
query = options.get("query")
try:
query = filter_queryset_xform_meta_perms_sql(xform, request.user,
query)
except NoRecordsPermission:
payload = {"details": _("You don't have permission")}
return Response(data=json.dumps(payload),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
else:
if query:
options['query'] = query
if export_type in EXTERNAL_EXPORT_TYPES and \
(token is not None) or (meta is not None):
export_type = Export.EXTERNAL_EXPORT
if export_type == Export.GOOGLE_SHEETS_EXPORT:
credential = _get_google_credential(request)
if isinstance(credential, HttpResponseRedirect):
return credential
options['google_credentials'] = credential.to_json()
if should_create_new_export(xform, export_type, options, request=request)\
or export_type == Export.EXTERNAL_EXPORT:
resp = {
u'job_uuid':
_create_export_async(xform,
export_type,
query,
False,
options=options)
}
else:
print('Do not create a new export.')
export = newest_export_for(xform, export_type, options)
if not export.filename:
# tends to happen when using newest_export_for.
resp = {
u'job_uuid':
_create_export_async(xform,
export_type,
query,
False,
options=options)
}
else:
resp = export_async_export_response(request, export)
return resp
def should_regenerate_export(xform, export_type, request):
return should_create_new_export(xform, export_type) or\
'start' in request.GET or 'end' in request.GET or\
'query' in request.GET or 'meta' in request.GET or\
'token' in request.GET
def data_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {
"xform": xform.id_string,
"export_type": export_type
}
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type) or query or\
'start' in request.GET or 'end' in request.GET:
# check for start and end params
if 'start' in request.GET or 'end' in request.GET:
if not query:
query = '{}'
query = json.dumps(
_set_submission_time_to_query(json.loads(query), request))
try:
export = generate_export(
export_type, extension, username, id_string, None, query)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
else:
export = newset_export_for(xform, export_type)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
if not export.filename:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
path, ext = os.path.splitext(export.filename)
ext = ext[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(
Export.EXPORT_MIMES[ext], id_string, extension=ext,
file_path=export.filepath)
return response
def export_list(request, username, id_string, export_type):
credential = None
if export_type not in Export.EXPORT_TYPE_DICT:
return HttpResponseBadRequest(
_(u'Export type "%s" is not supported.' % export_type))
if export_type == Export.GOOGLE_SHEETS_EXPORT:
# Retrieve google creds or redirect to google authorization page
credential = _get_google_credential(request)
if isinstance(credential, HttpResponseRedirect):
return credential
owner = get_object_or_404(User, username__iexact=username)
xform = get_form({'user': owner, 'id_string__iexact': id_string})
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if export_type == Export.EXTERNAL_EXPORT:
# check for template before trying to generate a report
if not MetaData.external_export(xform):
return HttpResponseForbidden(_(u'No XLS Template set.'))
# Get meta and token
export_token = request.GET.get('token')
export_meta = request.GET.get('meta')
options = {
'group_delimiter': DEFAULT_GROUP_DELIMITER,
'remove_group_name': False,
'split_select_multiples': True,
'binary_select_multiples': False,
'meta': export_meta,
'token': export_token,
'google_credentials': credential
}
if should_create_new_export(xform, export_type, options):
try:
create_async_export(xform,
export_type,
query=None,
force_xlsx=True,
options=options)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
metadata = MetaData.objects.filter(object_id=xform.id,
data_type="external_export")\
.values('id', 'data_value')
for m in metadata:
m['data_value'] = m.get('data_value').split('|')[0]
data = {
'username':
owner.username,
'xform':
xform,
'export_type':
export_type,
'export_type_name':
Export.EXPORT_TYPE_DICT[export_type],
'exports':
Export.objects.filter(xform=xform,
export_type=export_type).order_by('-created_on'),
'metas':
metadata
}
return render(request, 'export_list.html', data)
def data_export(request, username, id_string, export_type):
"""
Data export view.
"""
owner = get_object_or_404(User, username__iexact=username)
xform = get_form({'user': owner, 'id_string__iexact': id_string})
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {"xform": xform.id_string, "export_type": export_type}
options = {
"extension": extension,
"username": username,
"id_string": id_string
}
if query:
options['query'] = query
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type, options) or query or\
'start' in request.GET or 'end' in request.GET:
# check for start and end params
start, end = _get_start_end_submission_time(request)
options.update({"start": start, "end": end})
try:
export = generate_export(export_type, xform, None, options)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
except SPSSIOError as e:
return HttpResponseBadRequest(str(e))
else:
export = newest_export_for(xform, export_type, options)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") %
{'id_string': xform.id_string,
'export_type': export_type.upper()}, audit, request)
if not export.filename and not export.error_message:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
elif not export.filename and export.error_message:
return HttpResponseBadRequest(str(export.error_message))
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
__, extension = os.path.splitext(export.filename)
extension = extension[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(
Export.EXPORT_MIMES[extension],
id_string,
extension=extension,
file_path=export.filepath)
return response
def export_list(request, username, id_string, export_type):
"""
Export list view.
"""
credential = None
if export_type not in Export.EXPORT_TYPE_DICT:
return HttpResponseBadRequest(
_(u'Export type "%s" is not supported.' % export_type))
if export_type == Export.GOOGLE_SHEETS_EXPORT:
# Retrieve google creds or redirect to google authorization page
credential = _get_google_credential(request)
if isinstance(credential, HttpResponseRedirect):
return credential
owner = get_object_or_404(User, username__iexact=username)
xform = get_form({'user': owner, 'id_string__iexact': id_string})
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if export_type == Export.EXTERNAL_EXPORT:
# check for template before trying to generate a report
if not MetaData.external_export(xform):
return HttpResponseForbidden(_(u'No XLS Template set.'))
# Get meta and token
export_token = request.GET.get('token')
export_meta = request.GET.get('meta')
options = {
'group_delimiter': DEFAULT_GROUP_DELIMITER,
'remove_group_name': False,
'split_select_multiples': True,
'binary_select_multiples': False,
'meta': export_meta,
'token': export_token,
'google_credentials': credential
}
if should_create_new_export(xform, export_type, options):
try:
create_async_export(
xform,
export_type,
query=None,
force_xlsx=True,
options=options)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
metadata_qs = MetaData.objects.filter(object_id=xform.id,
data_type="external_export")\
.values('id', 'data_value')
for metadata in metadata_qs:
metadata['data_value'] = metadata.get('data_value').split('|')[0]
data = {
'username': owner.username,
'xform': xform,
'export_type': export_type,
'export_type_name': Export.EXPORT_TYPE_DICT[export_type],
'exports': Export.objects.filter(
xform=xform, export_type=export_type).order_by('-created_on'),
'metas': metadata_qs
} # yapf: disable
return render(request, 'export_list.html', data)
def should_regenerate_export(xform, export_type, request):
return should_create_new_export(xform, export_type) or\
'start' in request.GET or 'end' in request.GET or\
'query' in request.GET or 'meta' in request.GET or\
'token' in request.GET
def process_async_export(request, xform, export_type, options=None):
"""
Check if should generate export or just return the latest export.
Rules for regenerating an export are:
1. Filter included on the exports.
2. New submission done.
3. Always regenerate external exports.
(External exports uses templates and the template might have
changed)
:param request:
:param xform:
:param export_type:
:param options: additional export params that may include
query: export filter
token: template url for xls external reports
meta: metadataid that contains the external xls report template url
remove_group_name: Flag to determine if group names should appear
:return: response dictionary
"""
# maintain the order of keys while processing the export
export_type = _get_export_type(export_type)
token = options.get("token")
meta = options.get("meta")
query = options.get("query")
if export_type in external_export_types and \
(token is not None) or (meta is not None):
export_type = Export.EXTERNAL_EXPORT
dataview_pk = options.get('dataview_pk')
if export_type == Export.GOOGLE_SHEETS_EXPORT:
credential = _get_google_credential(request)
if isinstance(credential, HttpResponseRedirect):
return credential
options['google_credentials'] = credential
if should_create_new_export(xform, export_type, options, request=request)\
or export_type == Export.EXTERNAL_EXPORT:
resp = {
u'job_uuid':
_create_export_async(xform,
export_type,
query,
False,
options=options)
}
else:
export = newest_export_for(xform, export_type, options)
if not export.filename:
# tends to happen when using newest_export_for.
resp = {
u'job_uuid':
_create_export_async(xform,
export_type,
query,
False,
options=options)
}
else:
resp = _export_async_export_response(request,
xform,
export,
dataview_pk=dataview_pk)
return resp
def custom_response_handler(request,
xform,
query,
export_type,
token=None,
meta=None,
dataview=False,
filename=None):
"""
Returns a HTTP response with export file for download.
"""
export_type = _get_export_type(export_type)
if export_type in EXTERNAL_EXPORT_TYPES and \
(token is not None) or (meta is not None):
export_type = Export.EXTERNAL_EXPORT
options = parse_request_export_options(request.query_params)
dataview_pk = hasattr(dataview, 'pk') and dataview.pk
options["dataview_pk"] = dataview_pk
if dataview:
columns_with_hxl = get_columns_with_hxl(xform.survey.get('children'))
if columns_with_hxl:
options['include_hxl'] = include_hxl_row(dataview.columns,
list(columns_with_hxl))
try:
query = filter_queryset_xform_meta_perms_sql(xform, request.user,
query)
except NoRecordsPermission:
return Response(data=json.dumps(
{"details": _("You don't have permission")}),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
if query:
options['query'] = query
remove_group_name = options.get("remove_group_name")
export_id = request.query_params.get("export_id")
if export_id:
export = get_object_or_404(Export, id=export_id, xform=xform)
else:
if export_type == Export.GOOGLE_SHEETS_EXPORT:
return Response(data=json.dumps(
{"details": _("Sheets export only supported in async mode")}),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
# check if we need to re-generate,
# we always re-generate if a filter is specified
def _new_export():
return _generate_new_export(request,
xform,
query,
export_type,
dataview_pk=dataview_pk)
if should_create_new_export(xform,
export_type,
options,
request=request):
export = _new_export()
else:
export = newest_export_for(xform, export_type, options)
if not export.filename and not export.error_message:
export = _new_export()
log_export(request, xform, export_type)
if export_type == Export.EXTERNAL_EXPORT:
return external_export_response(export)
if export.filename is None and export.error_message:
raise exceptions.ParseError(export.error_message)
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
_path, ext = os.path.splitext(export.filename)
ext = ext[1:]
show_date = True
if filename is None and export.status == Export.SUCCESSFUL:
filename = _generate_filename(request,
xform,
remove_group_name,
dataview_pk=dataview_pk)
else:
show_date = False
response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext],
filename,
extension=ext,
show_date=show_date,
file_path=export.filepath)
return response
def data_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {
"xform": xform.id_string,
"export_type": export_type
}
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type) or query or\
'start' in request.GET or 'end' in request.GET:
format_date_for_mongo = lambda x, datetime: datetime.strptime(
x, '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S')
# check for start and end params
if 'start' in request.GET or 'end' in request.GET:
if not query:
query = '{}'
query = json.loads(query)
query[SUBMISSION_TIME] = {}
try:
if request.GET.get('start'):
query[SUBMISSION_TIME]['$gte'] = format_date_for_mongo(
request.GET['start'], datetime)
if request.GET.get('end'):
query[SUBMISSION_TIME]['$lte'] = format_date_for_mongo(
request.GET['end'], datetime)
except ValueError:
return HttpResponseBadRequest(
_("Dates must be in the format YY_MM_DD_hh_mm_ss"))
else:
query = json.dumps(query)
try:
export = generate_export(
export_type, extension, username, id_string, None, query)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
else:
export = newset_export_for(xform, export_type)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
if not export.filename:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
path, ext = os.path.splitext(export.filename)
ext = ext[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(
Export.EXPORT_MIMES[ext], id_string, extension=ext,
file_path=export.filepath)
return response
def data_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {
"xform": xform.id_string,
"export_type": export_type
}
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type) or query or\
'start' in request.GET or 'end' in request.GET:
format_date_for_mongo = lambda x, datetime: datetime.strptime(
x, '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S')
# check for start and end params
if 'start' in request.GET or 'end' in request.GET:
if not query:
query = '{}'
query = json.loads(query)
query[SUBMISSION_TIME] = {}
try:
if request.GET.get('start'):
query[SUBMISSION_TIME]['$gte'] = format_date_for_mongo(
request.GET['start'], datetime)
if request.GET.get('end'):
query[SUBMISSION_TIME]['$lte'] = format_date_for_mongo(
request.GET['end'], datetime)
except ValueError:
return HttpResponseBadRequest(
_("Dates must be in the format YY_MM_DD_hh_mm_ss"))
else:
query = json.dumps(query)
try:
export = generate_export(
export_type, extension, username, id_string, None, query)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
else:
export = newset_export_for(xform, export_type)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
if not export.filename:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
path, ext = os.path.splitext(export.filename)
ext = ext[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(
Export.EXPORT_MIMES[ext], id_string, extension=ext,
file_path=export.filepath)
return response
def data_export(request, username, id_string, export_type):
"""
Data export view.
"""
owner = get_object_or_404(User, username__iexact=username)
xform = get_form({'user': owner, 'id_string__iexact': id_string})
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {"xform": xform.id_string, "export_type": export_type}
options = {
"extension": extension,
"username": username,
"id_string": id_string
}
if query:
options['query'] = query
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type, options) or query or\
'start' in request.GET or 'end' in request.GET:
# check for start and end params
start, end = _get_start_end_submission_time(request)
options.update({"start": start, "end": end})
try:
export = generate_export(export_type, xform, None, options)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
except SPSSIOError as e:
return HttpResponseBadRequest(str(e))
else:
export = newest_export_for(xform, export_type, options)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
if not export.filename and not export.error_message:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
elif not export.filename and export.error_message:
return HttpResponseBadRequest(str(export.error_message))
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
__, extension = os.path.splitext(export.filename)
extension = extension[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(Export.EXPORT_MIMES[extension],
id_string,
extension=extension,
file_path=export.filepath)
return response
def data_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {"xform": xform.id_string, "export_type": export_type}
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type) or query or\
'start' in request.GET or 'end' in request.GET:
# check for start and end params
if 'start' in request.GET or 'end' in request.GET:
if not query:
query = '{}'
query = json.dumps(
_set_submission_time_to_query(json.loads(query), request))
try:
export = generate_export(export_type, extension, username,
id_string, None, query)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
else:
export = newset_export_for(xform, export_type)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
if not export.filename:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
path, ext = os.path.splitext(export.filename)
ext = ext[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext],
id_string,
extension=ext,
file_path=export.filepath)
return response
def export_list(request, username, id_string, export_type, is_project=0, id=0, site_id=0, version="0"):
site_id = int(site_id)
if export_type == Export.GDOC_EXPORT:
return HttpResponseForbidden(_(u'Not shared.'))
token = _get_google_token(request, redirect_url)
if isinstance(token, HttpResponse):
return token
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
if export_type == Export.EXTERNAL_EXPORT:
return HttpResponseForbidden(_(u'Not shared.'))
# check for template before trying to generate a report
if not MetaData.external_export(xform=xform):
return HttpResponseForbidden(_(u'No XLS Template set.'))
# Get meta and token
export_token = request.GET.get('token')
export_meta = request.GET.get('meta')
options = {
'meta': export_meta,
'token': export_token,
}
if should_create_new_export(xform, export_type, id, site_id, version):
if is_project == 1 or is_project == '1':
query = {"fs_project_uuid": str(id)}
else:
fsxf = FieldSightXF.objects.get(pk=id)
if fsxf.site:
query = {"fs_uuid": str(id)}
else:
query = {"fs_project_uuid": str(id), "fs_site": site_id}
force_xlsx = True
if version not in ["0", 0]:
query["__version__"] = version
deleted_at_query = {
"$or": [{"_deleted_at": {"$exists": False}},
{"_deleted_at": None}]}
# join existing query with deleted_at_query on an $and
query = {"$and": [query, deleted_at_query]}
print("query at excel generation", query)
try:
create_async_export(xform, export_type, query,
force_xlsx, options, is_project, id, site_id, version)
except Export.ExportTypeError:
return HttpResponseBadRequest(
_("%s is not a valid export type" % export_type))
metadata = MetaData.objects.filter(xform=xform,
data_type="external_export")\
.values('id', 'data_value')
for m in metadata:
m['data_value'] = m.get('data_value').split('|')[0]
data = {
'username': xform.user.username,
'xform': xform,
'export_type': export_type,
'export_type_name': Export.EXPORT_TYPE_DICT[export_type],
'exports': Export.objects.filter(
xform=xform, export_type=export_type, fsxf=id, site=site_id, version=version).order_by('-created_on'),
'metas': metadata,
'is_project': is_project,
'id': id,
'version': version,
}
data['site_id'] = site_id
return render(request, 'export_list.html', data)
