def export_list(request, username, id_string, export_type): if export_type == Export.GDOC_EXPORT: redirect_url = reverse( export_list, kwargs={ 'username': username, 'id_string': id_string, 'export_type': export_type}) token = _get_google_token(request, redirect_url) if isinstance(token, HttpResponse): return token owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, id_string__iexact=id_string, user=owner) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if should_create_new_export(xform, export_type): try: create_async_export( xform, export_type, query=None, force_xlsx=True) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) data = { 'username': owner.username, 'xform': xform, 'export_type': export_type, 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], 'exports': Export.objects.filter( xform=xform, export_type=export_type).order_by('-created_on') } return render(request, 'export_list.html', data)
def export_list(request, username, id_string, export_type): if export_type == Export.GDOC_EXPORT: redirect_url = reverse( export_list, kwargs={ 'username': username, 'id_string': id_string, 'export_type': export_type}) token = _get_google_token(request, redirect_url) if isinstance(token, HttpResponse): return token owner = get_object_or_404(User, username=username) xform = get_object_or_404(XForm, id_string=id_string, user=owner) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if should_create_new_export(xform, export_type): try: create_async_export( xform, export_type, query=None, force_xlsx=True) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) context = RequestContext(request) context.username = owner.username context.xform = xform # TODO: better output e.g. Excel instead of XLS context.export_type = export_type context.export_type_name = Export.EXPORT_TYPE_DICT[export_type] exports = Export.objects.filter(xform=xform, export_type=export_type)\ .order_by('-created_on') context.exports = exports return render_to_response('export_list.html', context_instance=context)
def export_list(request, username, id_string, export_type): if export_type == Export.GDOC_EXPORT: redirect_url = reverse( export_list, kwargs={ 'username': username, 'id_string': id_string, 'export_type': export_type}) token = _get_google_token(request, redirect_url) if isinstance(token, HttpResponse): return token owner = get_object_or_404(User, username=username) xform = get_object_or_404(XForm, id_string=id_string, user=owner) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if should_create_new_export(xform, export_type): try: create_async_export( xform, export_type, query=None, force_xlsx=True) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) context = RequestContext(request) context.username = owner.username context.xform = xform # TODO: better output e.g. Excel instead of XLS context.export_type = export_type context.export_type_name = Export.EXPORT_TYPE_DICT[export_type] exports = Export.objects.filter(xform=xform, export_type=export_type)\ .order_by('-created_on') context.exports = exports return render_to_response('export_list.html', context_instance=context)
def export_list(request, username, id_string, export_type): if export_type == Export.GDOC_EXPORT: redirect_url = reverse( export_list, kwargs={ 'username': username, 'id_string': id_string, 'export_type': export_type}) token = _get_google_token(request, redirect_url) if isinstance(token, HttpResponse): return token owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform=xform): return HttpResponseForbidden(_(u'No XLS Template set.')) # Get meta and token export_token = request.GET.get('token') export_meta = request.GET.get('meta') options = { 'meta': export_meta, 'token': export_token, } if should_create_new_export(xform, export_type): try: create_async_export( xform, export_type, query=None, force_xlsx=True, options=options) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) metadata = MetaData.objects.filter(xform=xform, data_type="external_export")\ .values('id', 'data_value') for m in metadata: m['data_value'] = m.get('data_value').split('|')[0] data = { 'username': owner.username, 'xform': xform, 'export_type': export_type, 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], 'exports': Export.objects.filter( xform=xform, export_type=export_type).order_by('-created_on'), 'metas': metadata } return render(request, 'export_list.html', data)
def test_should_create_new_export(self): # should only create new export if filter is defined # Test setup export_type = "csv" options = {"group_delimiter": "."} self._publish_transportation_form_and_submit_instance() will_create_new_export = should_create_new_export( self.xform, export_type, options) self.assertTrue(will_create_new_export)
def export_list(request, username, id_string, export_type): if export_type == Export.GDOC_EXPORT: redirect_url = reverse( export_list, kwargs={ 'username': username, 'id_string': id_string, 'export_type': export_type}) token = _get_google_token(request, redirect_url) if isinstance(token, HttpResponse): return token owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform=xform): return HttpResponseForbidden(_(u'No XLS Template set.')) # Get meta and token export_token = request.GET.get('token') export_meta = request.GET.get('meta') options = { 'meta': export_meta, 'token': export_token, } if should_create_new_export(xform, export_type): try: create_async_export( xform, export_type, query=None, force_xlsx=True, options=options) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) metadata = MetaData.objects.filter(xform=xform, data_type="external_export")\ .values('id', 'data_value') for m in metadata: m['data_value'] = m.get('data_value').split('|')[0] data = { 'username': owner.username, 'xform': xform, 'export_type': export_type, 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], 'exports': Export.objects.filter( xform=xform, export_type=export_type).order_by('-created_on'), 'metas': metadata } return render(request, 'export_list.html', data)
def test_should_create_new_export(self): # should only create new export if filter is defined # Test setup export_type = "csv" options = {"group_delimiter": "."} self._publish_transportation_form_and_submit_instance() will_create_new_export = should_create_new_export( self.xform, export_type, options) self.assertTrue(will_create_new_export)
def test_should_not_create_new_export_when_old_exists(self): export_type = "csv" self._publish_transportation_form_and_submit_instance() options = { "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True } self._create_old_export(self.xform, export_type, options) will_create_new_export = should_create_new_export( self.xform, export_type, options) self.assertFalse(will_create_new_export)
def test_should_not_create_new_export_when_old_exists(self): export_type = "csv" self._publish_transportation_form_and_submit_instance() options = { "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True } self._create_old_export(self.xform, export_type, options) will_create_new_export = should_create_new_export( self.xform, export_type, options) self.assertFalse(will_create_new_export)
def test_should_create_new_export_when_filter_defined(self): export_type = "csv" options = { "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) # Call should_create_new_export with updated options options['remove_group_name'] = True will_create_new_export = should_create_new_export( self.xform, export_type, options) self.assertTrue(will_create_new_export)
def test_should_create_new_export_when_filter_defined(self): export_type = "csv" options = { "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) # Call should_create_new_export with updated options options['remove_group_name'] = True will_create_new_export = should_create_new_export( self.xform, export_type, options) self.assertTrue(will_create_new_export)
def process_async_export(request, xform, export_type, options=None): """ Check if should generate export or just return the latest export. Rules for regenerating an export are: 1. Filter included on the exports. 2. New submission done. 3. Always regenerate external exports. (External exports uses templates and the template might have changed) :param request: :param xform: :param export_type: :param options: additional export params that may include query: export filter token: template url for xls external reports meta: metadataid that contains the external xls report template url remove_group_name: Flag to determine if group names should appear :return: response dictionary """ # maintain the order of keys while processing the export export_type = _get_export_type(export_type) token = options.get("token") meta = options.get("meta") query = options.get("query") try: query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: payload = {"details": _("You don't have permission")} return Response(data=json.dumps(payload), status=status.HTTP_403_FORBIDDEN, content_type="application/json") else: if query: options['query'] = query if export_type in EXTERNAL_EXPORT_TYPES and \ (token is not None) or (meta is not None): export_type = Export.EXTERNAL_EXPORT if export_type == Export.GOOGLE_SHEETS_EXPORT: credential = _get_google_credential(request) if isinstance(credential, HttpResponseRedirect): return credential options['google_credentials'] = credential.to_json() if should_create_new_export(xform, export_type, options, request=request)\ or export_type == Export.EXTERNAL_EXPORT: resp = { u'job_uuid': _create_export_async(xform, export_type, query, False, options=options) } else: print('Do not create a new export.') export = newest_export_for(xform, export_type, options) if not export.filename: # tends to happen when using newest_export_for. resp = { u'job_uuid': _create_export_async(xform, export_type, query, False, options=options) } else: resp = export_async_export_response(request, export) return resp
def should_regenerate_export(xform, export_type, request): return should_create_new_export(xform, export_type) or\ 'start' in request.GET or 'end' in request.GET or\ 'query' in request.GET or 'meta' in request.GET or\ 'token' in request.GET
def data_export(request, username, id_string, export_type): owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) query = request.GET.get("query") extension = export_type # check if we should force xlsx force_xlsx = request.GET.get('xls') != 'true' if export_type == Export.XLS_EXPORT and force_xlsx: extension = 'xlsx' elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: extension = 'zip' audit = { "xform": xform.id_string, "export_type": export_type } # check if we need to re-generate, # we always re-generate if a filter is specified if should_create_new_export(xform, export_type) or query or\ 'start' in request.GET or 'end' in request.GET: # check for start and end params if 'start' in request.GET or 'end' in request.GET: if not query: query = '{}' query = json.dumps( _set_submission_time_to_query(json.loads(query), request)) try: export = generate_export( export_type, extension, username, id_string, None, query) audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) else: export = newset_export_for(xform, export_type) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) if not export.filename: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") # get extension from file_path, exporter could modify to # xlsx if it exceeds limits path, ext = os.path.splitext(export.filename) ext = ext[1:] if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name( Export.EXPORT_MIMES[ext], id_string, extension=ext, file_path=export.filepath) return response
def export_list(request, username, id_string, export_type): credential = None if export_type not in Export.EXPORT_TYPE_DICT: return HttpResponseBadRequest( _(u'Export type "%s" is not supported.' % export_type)) if export_type == Export.GOOGLE_SHEETS_EXPORT: # Retrieve google creds or redirect to google authorization page credential = _get_google_credential(request) if isinstance(credential, HttpResponseRedirect): return credential owner = get_object_or_404(User, username__iexact=username) xform = get_form({'user': owner, 'id_string__iexact': id_string}) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): return HttpResponseForbidden(_(u'No XLS Template set.')) # Get meta and token export_token = request.GET.get('token') export_meta = request.GET.get('meta') options = { 'group_delimiter': DEFAULT_GROUP_DELIMITER, 'remove_group_name': False, 'split_select_multiples': True, 'binary_select_multiples': False, 'meta': export_meta, 'token': export_token, 'google_credentials': credential } if should_create_new_export(xform, export_type, options): try: create_async_export(xform, export_type, query=None, force_xlsx=True, options=options) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) metadata = MetaData.objects.filter(object_id=xform.id, data_type="external_export")\ .values('id', 'data_value') for m in metadata: m['data_value'] = m.get('data_value').split('|')[0] data = { 'username': owner.username, 'xform': xform, 'export_type': export_type, 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], 'exports': Export.objects.filter(xform=xform, export_type=export_type).order_by('-created_on'), 'metas': metadata } return render(request, 'export_list.html', data)
def data_export(request, username, id_string, export_type): """ Data export view. """ owner = get_object_or_404(User, username__iexact=username) xform = get_form({'user': owner, 'id_string__iexact': id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) query = request.GET.get("query") extension = export_type # check if we should force xlsx force_xlsx = request.GET.get('xls') != 'true' if export_type == Export.XLS_EXPORT and force_xlsx: extension = 'xlsx' elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: extension = 'zip' audit = {"xform": xform.id_string, "export_type": export_type} options = { "extension": extension, "username": username, "id_string": id_string } if query: options['query'] = query # check if we need to re-generate, # we always re-generate if a filter is specified if should_create_new_export(xform, export_type, options) or query or\ 'start' in request.GET or 'end' in request.GET: # check for start and end params start, end = _get_start_end_submission_time(request) options.update({"start": start, "end": end}) try: export = generate_export(export_type, xform, None, options) audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) except SPSSIOError as e: return HttpResponseBadRequest(str(e)) else: export = newest_export_for(xform, export_type, options) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded %(export_type)s export on '%(id_string)s'.") % {'id_string': xform.id_string, 'export_type': export_type.upper()}, audit, request) if not export.filename and not export.error_message: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") elif not export.filename and export.error_message: return HttpResponseBadRequest(str(export.error_message)) # get extension from file_path, exporter could modify to # xlsx if it exceeds limits __, extension = os.path.splitext(export.filename) extension = extension[1:] if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name( Export.EXPORT_MIMES[extension], id_string, extension=extension, file_path=export.filepath) return response
def export_list(request, username, id_string, export_type): """ Export list view. """ credential = None if export_type not in Export.EXPORT_TYPE_DICT: return HttpResponseBadRequest( _(u'Export type "%s" is not supported.' % export_type)) if export_type == Export.GOOGLE_SHEETS_EXPORT: # Retrieve google creds or redirect to google authorization page credential = _get_google_credential(request) if isinstance(credential, HttpResponseRedirect): return credential owner = get_object_or_404(User, username__iexact=username) xform = get_form({'user': owner, 'id_string__iexact': id_string}) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): return HttpResponseForbidden(_(u'No XLS Template set.')) # Get meta and token export_token = request.GET.get('token') export_meta = request.GET.get('meta') options = { 'group_delimiter': DEFAULT_GROUP_DELIMITER, 'remove_group_name': False, 'split_select_multiples': True, 'binary_select_multiples': False, 'meta': export_meta, 'token': export_token, 'google_credentials': credential } if should_create_new_export(xform, export_type, options): try: create_async_export( xform, export_type, query=None, force_xlsx=True, options=options) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) metadata_qs = MetaData.objects.filter(object_id=xform.id, data_type="external_export")\ .values('id', 'data_value') for metadata in metadata_qs: metadata['data_value'] = metadata.get('data_value').split('|')[0] data = { 'username': owner.username, 'xform': xform, 'export_type': export_type, 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], 'exports': Export.objects.filter( xform=xform, export_type=export_type).order_by('-created_on'), 'metas': metadata_qs } # yapf: disable return render(request, 'export_list.html', data)
def should_regenerate_export(xform, export_type, request): return should_create_new_export(xform, export_type) or\ 'start' in request.GET or 'end' in request.GET or\ 'query' in request.GET or 'meta' in request.GET or\ 'token' in request.GET
def process_async_export(request, xform, export_type, options=None): """ Check if should generate export or just return the latest export. Rules for regenerating an export are: 1. Filter included on the exports. 2. New submission done. 3. Always regenerate external exports. (External exports uses templates and the template might have changed) :param request: :param xform: :param export_type: :param options: additional export params that may include query: export filter token: template url for xls external reports meta: metadataid that contains the external xls report template url remove_group_name: Flag to determine if group names should appear :return: response dictionary """ # maintain the order of keys while processing the export export_type = _get_export_type(export_type) token = options.get("token") meta = options.get("meta") query = options.get("query") if export_type in external_export_types and \ (token is not None) or (meta is not None): export_type = Export.EXTERNAL_EXPORT dataview_pk = options.get('dataview_pk') if export_type == Export.GOOGLE_SHEETS_EXPORT: credential = _get_google_credential(request) if isinstance(credential, HttpResponseRedirect): return credential options['google_credentials'] = credential if should_create_new_export(xform, export_type, options, request=request)\ or export_type == Export.EXTERNAL_EXPORT: resp = { u'job_uuid': _create_export_async(xform, export_type, query, False, options=options) } else: export = newest_export_for(xform, export_type, options) if not export.filename: # tends to happen when using newest_export_for. resp = { u'job_uuid': _create_export_async(xform, export_type, query, False, options=options) } else: resp = _export_async_export_response(request, xform, export, dataview_pk=dataview_pk) return resp
def custom_response_handler(request, xform, query, export_type, token=None, meta=None, dataview=False, filename=None): """ Returns a HTTP response with export file for download. """ export_type = _get_export_type(export_type) if export_type in EXTERNAL_EXPORT_TYPES and \ (token is not None) or (meta is not None): export_type = Export.EXTERNAL_EXPORT options = parse_request_export_options(request.query_params) dataview_pk = hasattr(dataview, 'pk') and dataview.pk options["dataview_pk"] = dataview_pk if dataview: columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) if columns_with_hxl: options['include_hxl'] = include_hxl_row(dataview.columns, list(columns_with_hxl)) try: query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: return Response(data=json.dumps( {"details": _("You don't have permission")}), status=status.HTTP_403_FORBIDDEN, content_type="application/json") if query: options['query'] = query remove_group_name = options.get("remove_group_name") export_id = request.query_params.get("export_id") if export_id: export = get_object_or_404(Export, id=export_id, xform=xform) else: if export_type == Export.GOOGLE_SHEETS_EXPORT: return Response(data=json.dumps( {"details": _("Sheets export only supported in async mode")}), status=status.HTTP_403_FORBIDDEN, content_type="application/json") # check if we need to re-generate, # we always re-generate if a filter is specified def _new_export(): return _generate_new_export(request, xform, query, export_type, dataview_pk=dataview_pk) if should_create_new_export(xform, export_type, options, request=request): export = _new_export() else: export = newest_export_for(xform, export_type, options) if not export.filename and not export.error_message: export = _new_export() log_export(request, xform, export_type) if export_type == Export.EXTERNAL_EXPORT: return external_export_response(export) if export.filename is None and export.error_message: raise exceptions.ParseError(export.error_message) # get extension from file_path, exporter could modify to # xlsx if it exceeds limits _path, ext = os.path.splitext(export.filename) ext = ext[1:] show_date = True if filename is None and export.status == Export.SUCCESSFUL: filename = _generate_filename(request, xform, remove_group_name, dataview_pk=dataview_pk) else: show_date = False response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext], filename, extension=ext, show_date=show_date, file_path=export.filepath) return response
def data_export(request, username, id_string, export_type): owner = get_object_or_404(User, username=username) xform = get_object_or_404(XForm, id_string=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) query = request.GET.get("query") extension = export_type # check if we should force xlsx force_xlsx = request.GET.get('xls') != 'true' if export_type == Export.XLS_EXPORT and force_xlsx: extension = 'xlsx' elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: extension = 'zip' audit = { "xform": xform.id_string, "export_type": export_type } # check if we need to re-generate, # we always re-generate if a filter is specified if should_create_new_export(xform, export_type) or query or\ 'start' in request.GET or 'end' in request.GET: format_date_for_mongo = lambda x, datetime: datetime.strptime( x, '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S') # check for start and end params if 'start' in request.GET or 'end' in request.GET: if not query: query = '{}' query = json.loads(query) query[SUBMISSION_TIME] = {} try: if request.GET.get('start'): query[SUBMISSION_TIME]['$gte'] = format_date_for_mongo( request.GET['start'], datetime) if request.GET.get('end'): query[SUBMISSION_TIME]['$lte'] = format_date_for_mongo( request.GET['end'], datetime) except ValueError: return HttpResponseBadRequest( _("Dates must be in the format YY_MM_DD_hh_mm_ss")) else: query = json.dumps(query) try: export = generate_export( export_type, extension, username, id_string, None, query) audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) else: export = newset_export_for(xform, export_type) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) if not export.filename: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") # get extension from file_path, exporter could modify to # xlsx if it exceeds limits path, ext = os.path.splitext(export.filename) ext = ext[1:] if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name( Export.EXPORT_MIMES[ext], id_string, extension=ext, file_path=export.filepath) return response
def data_export(request, username, id_string, export_type): owner = get_object_or_404(User, username=username) xform = get_object_or_404(XForm, id_string=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) query = request.GET.get("query") extension = export_type # check if we should force xlsx force_xlsx = request.GET.get('xls') != 'true' if export_type == Export.XLS_EXPORT and force_xlsx: extension = 'xlsx' elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: extension = 'zip' audit = { "xform": xform.id_string, "export_type": export_type } # check if we need to re-generate, # we always re-generate if a filter is specified if should_create_new_export(xform, export_type) or query or\ 'start' in request.GET or 'end' in request.GET: format_date_for_mongo = lambda x, datetime: datetime.strptime( x, '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S') # check for start and end params if 'start' in request.GET or 'end' in request.GET: if not query: query = '{}' query = json.loads(query) query[SUBMISSION_TIME] = {} try: if request.GET.get('start'): query[SUBMISSION_TIME]['$gte'] = format_date_for_mongo( request.GET['start'], datetime) if request.GET.get('end'): query[SUBMISSION_TIME]['$lte'] = format_date_for_mongo( request.GET['end'], datetime) except ValueError: return HttpResponseBadRequest( _("Dates must be in the format YY_MM_DD_hh_mm_ss")) else: query = json.dumps(query) try: export = generate_export( export_type, extension, username, id_string, None, query) audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) else: export = newset_export_for(xform, export_type) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) if not export.filename: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") # get extension from file_path, exporter could modify to # xlsx if it exceeds limits path, ext = os.path.splitext(export.filename) ext = ext[1:] if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name( Export.EXPORT_MIMES[ext], id_string, extension=ext, file_path=export.filepath) return response
def data_export(request, username, id_string, export_type): """ Data export view. """ owner = get_object_or_404(User, username__iexact=username) xform = get_form({'user': owner, 'id_string__iexact': id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) query = request.GET.get("query") extension = export_type # check if we should force xlsx force_xlsx = request.GET.get('xls') != 'true' if export_type == Export.XLS_EXPORT and force_xlsx: extension = 'xlsx' elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: extension = 'zip' audit = {"xform": xform.id_string, "export_type": export_type} options = { "extension": extension, "username": username, "id_string": id_string } if query: options['query'] = query # check if we need to re-generate, # we always re-generate if a filter is specified if should_create_new_export(xform, export_type, options) or query or\ 'start' in request.GET or 'end' in request.GET: # check for start and end params start, end = _get_start_end_submission_time(request) options.update({"start": start, "end": end}) try: export = generate_export(export_type, xform, None, options) audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) except SPSSIOError as e: return HttpResponseBadRequest(str(e)) else: export = newest_export_for(xform, export_type, options) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) if not export.filename and not export.error_message: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") elif not export.filename and export.error_message: return HttpResponseBadRequest(str(export.error_message)) # get extension from file_path, exporter could modify to # xlsx if it exceeds limits __, extension = os.path.splitext(export.filename) extension = extension[1:] if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name(Export.EXPORT_MIMES[extension], id_string, extension=extension, file_path=export.filepath) return response
def data_export(request, username, id_string, export_type): owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) query = request.GET.get("query") extension = export_type # check if we should force xlsx force_xlsx = request.GET.get('xls') != 'true' if export_type == Export.XLS_EXPORT and force_xlsx: extension = 'xlsx' elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: extension = 'zip' audit = {"xform": xform.id_string, "export_type": export_type} # check if we need to re-generate, # we always re-generate if a filter is specified if should_create_new_export(xform, export_type) or query or\ 'start' in request.GET or 'end' in request.GET: # check for start and end params if 'start' in request.GET or 'end' in request.GET: if not query: query = '{}' query = json.dumps( _set_submission_time_to_query(json.loads(query), request)) try: export = generate_export(export_type, extension, username, id_string, None, query) audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) else: export = newset_export_for(xform, export_type) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) if not export.filename: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") # get extension from file_path, exporter could modify to # xlsx if it exceeds limits path, ext = os.path.splitext(export.filename) ext = ext[1:] if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext], id_string, extension=ext, file_path=export.filepath) return response
def export_list(request, username, id_string, export_type, is_project=0, id=0, site_id=0, version="0"): site_id = int(site_id) if export_type == Export.GDOC_EXPORT: return HttpResponseForbidden(_(u'Not shared.')) token = _get_google_token(request, redirect_url) if isinstance(token, HttpResponse): return token owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner) if export_type == Export.EXTERNAL_EXPORT: return HttpResponseForbidden(_(u'Not shared.')) # check for template before trying to generate a report if not MetaData.external_export(xform=xform): return HttpResponseForbidden(_(u'No XLS Template set.')) # Get meta and token export_token = request.GET.get('token') export_meta = request.GET.get('meta') options = { 'meta': export_meta, 'token': export_token, } if should_create_new_export(xform, export_type, id, site_id, version): if is_project == 1 or is_project == '1': query = {"fs_project_uuid": str(id)} else: fsxf = FieldSightXF.objects.get(pk=id) if fsxf.site: query = {"fs_uuid": str(id)} else: query = {"fs_project_uuid": str(id), "fs_site": site_id} force_xlsx = True if version not in ["0", 0]: query["__version__"] = version deleted_at_query = { "$or": [{"_deleted_at": {"$exists": False}}, {"_deleted_at": None}]} # join existing query with deleted_at_query on an $and query = {"$and": [query, deleted_at_query]} print("query at excel generation", query) try: create_async_export(xform, export_type, query, force_xlsx, options, is_project, id, site_id, version) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) metadata = MetaData.objects.filter(xform=xform, data_type="external_export")\ .values('id', 'data_value') for m in metadata: m['data_value'] = m.get('data_value').split('|')[0] data = { 'username': xform.user.username, 'xform': xform, 'export_type': export_type, 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], 'exports': Export.objects.filter( xform=xform, export_type=export_type, fsxf=id, site=site_id, version=version).order_by('-created_on'), 'metas': metadata, 'is_project': is_project, 'id': id, 'version': version, } data['site_id'] = site_id return render(request, 'export_list.html', data)