def prepare_form_multimedia(request, domain):
"""Gets the download_id for the multimedia zip and sends it to the
exportDownloadService in download_export.ng.js to begin polling for the
zip file download.
"""
form_or_case = request.POST.get('form_or_case')
sms_export = json.loads(request.POST.get('sms_export'))
permissions = ExportsPermissionsManager(form_or_case, domain, request.couch_user)
permissions.access_download_export_or_404()
view_helper = DownloadExportViewHelper.get(request, domain, form_or_case, sms_export)
filter_form_data = json.loads(request.POST.get('form_data'))
export_specs = json.loads(request.POST.get('exports'))
try:
filter_form = view_helper.get_filter_form(filter_form_data)
except ExportFormValidationException:
return json_response({
'error': _("Please check that you've submitted all required filters."),
})
download = DownloadBase()
export_object = view_helper.get_export(export_specs[0]['export_id'])
task_kwargs = filter_form.get_multimedia_task_kwargs(export_object, download.download_id, filter_form_data)
from corehq.apps.reports.tasks import build_form_multimedia_zip
download.set_task(build_form_multimedia_zip.delay(**task_kwargs))
return json_response({
'success': True,
'download_id': download.download_id,
})
def prepare_form_multimedia(request, domain):
"""Gets the download_id for the multimedia zip and sends it to the
exportDownloadService in download_export.ng.js to begin polling for the
zip file download.
"""
form_or_case = request.POST.get('form_or_case')
sms_export = json.loads(request.POST.get('sms_export'))
permissions = ExportsPermissionsManager(form_or_case, domain,
request.couch_user)
permissions.access_download_export_or_404()
view_helper = DownloadExportViewHelper.get(request, domain, form_or_case,
sms_export)
filter_form_data = json.loads(request.POST.get('form_data'))
export_specs = json.loads(request.POST.get('exports'))
try:
filter_form = view_helper.get_filter_form(filter_form_data)
except ExportFormValidationException:
return JsonResponse({
'error':
_("Please check that you've submitted all required filters."),
})
download = DownloadBase()
export_object = view_helper.get_export(export_specs[0]['export_id'])
task_kwargs = filter_form.get_multimedia_task_kwargs(
export_object, download.download_id, filter_form_data)
from corehq.apps.reports.tasks import build_form_multimedia_zip
download.set_task(build_form_multimedia_zip.delay(**task_kwargs))
return JsonResponse({
'success': True,
'download_id': download.download_id,
})
def prepare_custom_export(request, domain):
"""Uses the current exports download framework (with some nasty filters)
to return the current download id to POLL for the download status.
:return: {
'success': True,
'download_id': '<some uuid>',
}
"""
form_or_case = request.POST.get('form_or_case')
sms_export = json.loads(request.POST.get('sms_export'))
permissions = ExportsPermissionsManager(form_or_case, domain, request.couch_user)
permissions.access_download_export_or_404()
view_helper = DownloadExportViewHelper.get(request, domain, form_or_case, sms_export)
filter_form_data = json.loads(request.POST.get('form_data'))
try:
filter_form = view_helper.get_filter_form(filter_form_data)
except ExportFormValidationException:
return json_response({
'error': _("Form did not validate."),
})
export_filters = filter_form.get_export_filters(request, filter_form_data)
export_es_filters = [f.to_es_filter() for f in export_filters]
export_specs = json.loads(request.POST.get('exports'))
export_ids = [spec['export_id'] for spec in export_specs]
export_instances = [view_helper.get_export(export_id) for export_id in export_ids]
try:
_check_deid_permissions(permissions, export_instances)
_check_export_size(domain, export_instances, export_filters)
except ExportAsyncException as e:
return json_response({
'error': str(e),
})
# Generate filename
if len(export_instances) > 1:
filename = "{}_custom_bulk_export_{}".format(domain, date.today().isoformat())
else:
filename = "{} {}".format(export_instances[0].name, date.today().isoformat())
download = get_export_download(
domain,
export_ids,
view_helper.model,
request.couch_user.username,
es_filters=export_es_filters,
owner_id=request.couch_user.get_id,
filename=filename,
)
view_helper.send_preparation_analytics(export_instances, export_filters)
return json_response({
'success': True,
'download_id': download.download_id,
})
def prepare_custom_export(request, domain):
"""Uses the current exports download framework (with some nasty filters)
to return the current download id to POLL for the download status.
:return: {
'success': True,
'download_id': '<some uuid>',
}
"""
form_or_case = request.POST.get('form_or_case')
sms_export = json.loads(request.POST.get('sms_export'))
permissions = ExportsPermissionsManager(form_or_case, domain, request.couch_user)
permissions.access_download_export_or_404()
view_helper = DownloadExportViewHelper.get(request, domain, form_or_case, sms_export)
filter_form_data = json.loads(request.POST.get('form_data'))
try:
filter_form = view_helper.get_filter_form(filter_form_data)
except ExportFormValidationException:
return json_response({
'error': _("Form did not validate."),
})
export_filters = filter_form.get_export_filters(request, filter_form_data)
export_specs = json.loads(request.POST.get('exports'))
export_ids = [spec['export_id'] for spec in export_specs]
export_instances = [view_helper.get_export(export_id) for export_id in export_ids]
try:
_check_deid_permissions(permissions, export_instances)
_check_export_size(domain, export_instances, export_filters)
except ExportAsyncException as e:
return json_response({
'error': six.text_type(e),
})
# Generate filename
if len(export_instances) > 1:
filename = "{}_custom_bulk_export_{}".format(domain, date.today().isoformat())
else:
filename = "{} {}".format(export_instances[0].name, date.today().isoformat())
download = get_export_download(
domain,
export_ids,
view_helper.model,
request.couch_user.username,
filters=export_filters,
filename=filename,
)
view_helper.send_preparation_analytics(export_instances, export_filters)
return json_response({
'success': True,
'download_id': download.download_id,
})
def prepare_form_multimedia(request, domain):
"""Gets the download_id for the multimedia zip and sends it to the
exportDownloadService in download_export.ng.js to begin polling for the
zip file download.
"""
form_or_case = request.POST.get('form_or_case')
sms_export = json.loads(request.POST.get('sms_export'))
permissions = ExportsPermissionsManager(form_or_case, domain,
request.couch_user)
permissions.access_download_export_or_404()
view_helper = DownloadExportViewHelper.get(request, domain, form_or_case,
sms_export)
filter_form_data = json.loads(request.POST.get('form_data'))
export_specs = json.loads(request.POST.get('exports'))
try:
filter_form = view_helper.get_filter_form(filter_form_data)
except ExportFormValidationException:
return json_response({
'error':
_("Please check that you've submitted all required filters."),
})
export = view_helper.get_export(export_specs[0]['export_id'])
datespan = filter_form.cleaned_data['date_range']
user_types = filter_form.get_es_user_types(filter_form_data)
if media_export_is_too_big(domain, export.app_id, export.xmlns, datespan,
user_types):
return json_response({
'success':
False,
'error':
_("This is too many files to export at once. "
"Please modify your filters to select fewer forms."),
})
download = DownloadBase()
download.set_task(
build_form_multimedia_zip.delay(
domain=domain,
export_id=export.get_id,
datespan=datespan,
user_types=user_types,
download_id=download.download_id,
owner_id=request.couch_user.get_id,
))
return json_response({
'success': True,
'download_id': download.download_id,
})
def has_multimedia(request, domain):
"""Checks to see if this form export has multimedia available to export
"""
form_or_case = request.GET.get('form_or_case')
if form_or_case != 'form':
raise ValueError("has_multimedia is only available for form exports")
permissions = ExportsPermissionsManager(form_or_case, domain, request.couch_user)
permissions.access_download_export_or_404()
export_object = FormExportInstance.get(request.GET.get('export_id'))
return json_response({
'success': True,
'hasMultimedia': export_object.has_multimedia,
})
def poll_custom_export_download(request, domain):
"""Polls celery to see how the export download task is going.
:return: final response: {
'success': True,
'dropbox_url': '<url>',
'download_url: '<url>',
<task info>
}
"""
form_or_case = request.GET.get('form_or_case')
permissions = ExportsPermissionsManager(form_or_case, domain,
request.couch_user)
permissions.access_download_export_or_404()
download_id = request.GET.get('download_id')
try:
context = get_download_context(download_id)
except TaskFailedError as e:
if e.exception_name == 'XlsLengthException':
return JsonResponse({
'error':
_('This file has more than 256 columns, which is not supported by xls. '
'Please change the output type to csv or xlsx to export this file.'
)
})
else:
notify_exception(request,
"Export download failed",
details={
'download_id': download_id,
'errors': e.errors,
'exception_name': e.exception_name
})
return JsonResponse({
'error': _("Download task failed to start."),
})
if context.get('is_ready', False):
context.update({
'dropbox_url':
reverse('dropbox_upload', args=(download_id, )),
'download_url':
"{}?get_file".format(
reverse('retrieve_download', args=(download_id, ))),
})
context['is_poll_successful'] = True
return json_response(context)
def has_multimedia(request, domain):
"""Checks to see if this form export has multimedia available to export
"""
form_or_case = request.GET.get('form_or_case')
if form_or_case != 'form':
raise ValueError("has_multimedia is only available for form exports")
permissions = ExportsPermissionsManager(form_or_case, domain,
request.couch_user)
permissions.access_download_export_or_404()
export_object = FormExportInstance.get(request.GET.get('export_id'))
if isinstance(export_object, ExportInstance):
has_multimedia = export_object.has_multimedia
else:
has_multimedia = forms_have_multimedia(
domain, export_object.app_id, getattr(export_object, 'xmlns', ''))
return JsonResponse({
'success': True,
'hasMultimedia': has_multimedia,
})
def poll_custom_export_download(request, domain):
"""Polls celery to see how the export download task is going.
:return: final response: {
'success': True,
'dropbox_url': '<url>',
'download_url: '<url>',
<task info>
}
"""
form_or_case = request.GET.get('form_or_case')
permissions = ExportsPermissionsManager(form_or_case, domain, request.couch_user)
permissions.access_download_export_or_404()
download_id = request.GET.get('download_id')
try:
context = get_download_context(download_id)
except TaskFailedError as e:
if e.exception_name == 'XlsLengthException':
return JsonResponse({
'error': _(
'This file has more than 256 columns, which is not supported by xls. '
'Please change the output type to csv or xlsx to export this file.')
})
else:
notify_exception(
request, "Export download failed",
details={'download_id': download_id, 'errors': e.errors,
'exception_name': e.exception_name})
return JsonResponse({
'error': _("Download task failed to start."),
})
if context.get('is_ready', False):
context.update({
'dropbox_url': reverse('dropbox_upload', args=(download_id,)),
'download_url': "{}?get_file".format(
reverse('retrieve_download', args=(download_id,))
),
})
context['is_poll_successful'] = True
return json_response(context)
def has_multimedia(request, domain):
"""Checks to see if this form export has multimedia available to export
"""
form_or_case = request.GET.get('form_or_case')
if form_or_case != 'form':
raise ValueError("has_multimedia is only available for form exports")
permissions = ExportsPermissionsManager(form_or_case, domain, request.couch_user)
permissions.access_download_export_or_404()
export_object = FormExportInstance.get(request.GET.get('export_id'))
if isinstance(export_object, ExportInstance):
has_multimedia = export_object.has_multimedia
else:
has_multimedia = forms_have_multimedia(
domain,
export_object.app_id,
getattr(export_object, 'xmlns', '')
)
return json_response({
'success': True,
'hasMultimedia': has_multimedia,
})
def poll_custom_export_download(request, domain):
"""Polls celery to see how the export download task is going.
:return: final response: {
'success': True,
'dropbox_url': '<url>',
'download_url: '<url>',
<task info>
}
"""
form_or_case = request.GET.get('form_or_case')
permissions = ExportsPermissionsManager(form_or_case, domain,
request.couch_user)
permissions.access_download_export_or_404()
download_id = request.GET.get('download_id')
try:
context = get_download_context(download_id)
except TaskFailedError as e:
notify_exception(request,
"Export download failed",
details={
'download_id': download_id,
'errors': e.errors
})
return JsonResponse({
'error': _("Download task failed to start."),
})
if context.get('is_ready', False):
context.update({
'dropbox_url':
reverse('dropbox_upload', args=(download_id, )),
'download_url':
"{}?get_file".format(
reverse('retrieve_download', args=(download_id, ))),
})
context['is_poll_successful'] = True
return JsonResponse(context)
def prepare_custom_export(request, domain):
"""Uses the current exports download framework (with some nasty filters)
to return the current download id to POLL for the download status.
:return: {
'success': True,
'download_id': '<some uuid>',
}
"""
form_or_case = request.POST.get('form_or_case')
sms_export = json.loads(request.POST.get('sms_export'))
permissions = ExportsPermissionsManager(form_or_case, domain,
request.couch_user)
permissions.access_download_export_or_404()
view_helper = DownloadExportViewHelper.get(request, domain, form_or_case,
sms_export)
filter_form_data = json.loads(request.POST.get('form_data'))
try:
filter_form = view_helper.get_filter_form(filter_form_data)
except ExportFormValidationException:
return JsonResponse({
'error': _("Form did not validate."),
})
export_filters = filter_form.get_export_filters(request, filter_form_data)
export_specs = json.loads(request.POST.get('exports'))
export_ids = [spec['export_id'] for spec in export_specs]
export_instances = [
view_helper.get_export(export_id) for export_id in export_ids
]
try:
_check_deid_permissions(permissions, export_instances)
_check_export_size(domain, export_instances, export_filters)
except ExportAsyncException as e:
return JsonResponse({
'error': six.text_type(e),
})
# Generate filename
if len(export_instances) > 1:
filename = "{}_custom_bulk_export_{}".format(domain,
date.today().isoformat())
else:
filename = "{} {}".format(export_instances[0].name,
date.today().isoformat())
try:
download = get_export_download(
domain,
export_ids,
view_helper.model,
request.couch_user.username,
filters=export_filters,
filename=filename,
)
except XlsLengthException:
return JsonResponse({
'error':
_('This file has more than 256 columns, which is not supported by xls. '
'Please change the output type to csv or xlsx to export this file.'
)
})
view_helper.send_preparation_analytics(export_instances, export_filters)
return JsonResponse({
'success': True,
'download_id': download.download_id,
})
class BaseDownloadExportView(HQJSONResponseMixin, BaseProjectDataView):
template_name = 'export/download_export.html'
http_method_names = ['get', 'post']
show_date_range = False
check_for_multimedia = False
sms_export = False
# To serve filters for export from mobile_user_and_group_slugs
export_filter_class = None
@use_daterangepicker
@use_select2
@method_decorator(login_and_domain_required)
def dispatch(self, request, *args, **kwargs):
self.permissions = ExportsPermissionsManager(self.form_or_case,
request.domain,
request.couch_user)
self.permissions.access_download_export_or_404()
return super(BaseDownloadExportView,
self).dispatch(request, *args, **kwargs)
def post(self, request, *args, **kwargs):
if not request.is_ajax():
context = self.get_context_data(**kwargs)
return self.render_to_response(context)
return super(BaseDownloadExportView,
self).post(request, *args, **kwargs)
@property
@memoized
def view_helper(self):
return DownloadExportViewHelper.get(self.request, self.domain,
self.form_or_case, self.sms_export)
@property
@memoized
def timezone(self):
return get_timezone(self.domain, self.request.couch_user)
@property
@memoized
def default_datespan(self):
return datespan_from_beginning(self.domain_object, self.timezone)
@property
def page_context(self):
context = {
'download_export_form': self.download_export_form,
'export_list': self.export_list,
'form_or_case': self.form_or_case,
'max_column_size': self.max_column_size,
'show_date_range': self.show_date_range,
'check_for_multimedia': self.check_for_multimedia,
'sms_export': self.sms_export,
'user_types': HQUserType.human_readable
}
if (self.default_datespan.startdate is not None
and self.default_datespan.enddate is not None):
context.update({
'default_date_range':
'{startdate}{separator}{enddate}'.format(
startdate=self.default_datespan.startdate.strftime(
'%Y-%m-%d'),
enddate=self.default_datespan.enddate.strftime('%Y-%m-%d'),
separator=DateRangePickerWidget.separator,
),
})
else:
context.update({
'default_date_range':
_("You have no submissions in this project."),
'show_no_submissions_warning':
True,
})
if self.export_filter_class:
context['dynamic_filters'] = self.export_filter_class(
self.request, self.request.domain).render()
return context
@property
@memoized
def download_export_form(self):
return self.view_helper.filter_form_class(self.domain_object,
timezone=self.timezone)
@property
def export_id(self):
return self.kwargs.get('export_id')
@property
def page_url(self):
if self.export_id:
return reverse(self.urlname, args=(self.domain, self.export_id))
return reverse(self.urlname, args=(self.domain, ))
@property
def export_list(self):
exports = []
if (self.request.method == 'POST'
and 'export_list' in self.request.POST
and not self.request.is_ajax()):
raw_export_list = json.loads(self.request.POST['export_list'])
exports = [
self.view_helper.get_export(e['id']) for e in raw_export_list
]
elif self.export_id or self.sms_export:
exports = [self.view_helper.get_export(self.export_id)]
if not self.permissions.has_view_permissions:
if self.permissions.has_deid_view_permissions:
exports = [x for x in exports if x.is_safe]
else:
raise Http404()
# if there are no exports, this page doesn't exist
if not exports:
raise Http404()
exports = [
self.download_export_form.format_export_data(e) for e in exports
]
return exports
@property
def max_column_size(self):
try:
return int(self.request.GET.get('max_column_size', 2000))
except TypeError:
return 2000
class BaseDownloadExportView(HQJSONResponseMixin, BaseProjectDataView):
template_name = 'export/download_export.html'
http_method_names = ['get', 'post']
show_date_range = False
check_for_multimedia = False
sms_export = False
# To serve filters for export from mobile_user_and_group_slugs
export_filter_class = None
@use_daterangepicker
@use_select2
@method_decorator(login_and_domain_required)
def dispatch(self, request, *args, **kwargs):
self.permissions = ExportsPermissionsManager(self.form_or_case, request.domain, request.couch_user)
self.permissions.access_download_export_or_404()
return super(BaseDownloadExportView, self).dispatch(request, *args, **kwargs)
def post(self, request, *args, **kwargs):
if not request.is_ajax():
context = self.get_context_data(**kwargs)
return self.render_to_response(context)
return super(BaseDownloadExportView, self).post(request, *args, **kwargs)
@property
@memoized
def view_helper(self):
return DownloadExportViewHelper.get(self.request, self.domain, self.form_or_case, self.sms_export)
@property
@memoized
def timezone(self):
return get_timezone(self.domain, self.request.couch_user)
@property
@memoized
def default_datespan(self):
return datespan_from_beginning(self.domain_object, self.timezone)
@property
def page_context(self):
context = {
'download_export_form': self.download_export_form,
'export_list': self.export_list,
'form_or_case': self.form_or_case,
'max_column_size': self.max_column_size,
'show_date_range': self.show_date_range,
'check_for_multimedia': self.check_for_multimedia,
'sms_export': self.sms_export,
'user_types': HQUserType.human_readable
}
if (
self.default_datespan.startdate is not None
and self.default_datespan.enddate is not None
):
context.update({
'default_date_range': '{startdate}{separator}{enddate}'.format(
startdate=self.default_datespan.startdate.strftime('%Y-%m-%d'),
enddate=self.default_datespan.enddate.strftime('%Y-%m-%d'),
separator=DateRangePickerWidget.separator,
),
})
else:
context.update({
'default_date_range': _(
"You have no submissions in this project."
),
'show_no_submissions_warning': True,
})
if self.export_filter_class:
context['dynamic_filters'] = self.export_filter_class(
self.request, self.request.domain
).render()
return context
@property
@memoized
def download_export_form(self):
return self.view_helper.filter_form_class(self.domain_object, timezone=self.timezone)
@property
def export_id(self):
return self.kwargs.get('export_id')
@property
def page_url(self):
if self.export_id:
return reverse(self.urlname, args=(self.domain, self.export_id))
return reverse(self.urlname, args=(self.domain,))
@property
def export_list(self):
exports = []
if (
self.request.method == 'POST'
and 'export_list' in self.request.POST
and not self.request.is_ajax()
):
raw_export_list = json.loads(self.request.POST['export_list'])
exports = [self.view_helper.get_export(e['id']) for e in raw_export_list]
elif self.export_id or self.sms_export:
exports = [self.view_helper.get_export(self.export_id)]
if not self.permissions.has_view_permissions:
if self.permissions.has_deid_view_permissions:
exports = [x for x in exports if x.is_safe]
else:
raise Http404()
# if there are no exports, this page doesn't exist
if not exports:
raise Http404()
exports = [self.download_export_form.format_export_data(e) for e in exports]
return exports
@property
def max_column_size(self):
try:
return int(self.request.GET.get('max_column_size', 2000))
except TypeError:
return 2000
