def export_async(self, request, *args, **kwargs): params = request.query_params job_uuid = params.get('job_uuid') export_type = params.get('format') include_hxl = params.get('include_hxl', False) include_labels = params.get('include_labels', False) include_labels_only = params.get('include_labels_only', False) query = params.get("query") dataview = self.get_object() xform = dataview.xform if include_labels is not None: include_labels = str_to_bool(include_labels) if include_labels_only is not None: include_labels_only = str_to_bool(include_labels_only) if include_hxl is not None: include_hxl = str_to_bool(include_hxl) remove_group_name = params.get('remove_group_name', False) columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) if columns_with_hxl and include_hxl: include_hxl = include_hxl_row( dataview.columns, list(columns_with_hxl) ) options = { 'remove_group_name': remove_group_name, 'dataview_pk': dataview.pk, 'include_hxl': include_hxl, 'include_labels': include_labels, 'include_labels_only': include_labels_only } if query: options.update({'query': query}) if job_uuid: job = AsyncResult(job_uuid) if job.state == 'SUCCESS': export_id = job.result export = Export.objects.get(id=export_id) resp = export_async_export_response(request, export) else: resp = { 'job_status': job.state } else: resp = process_async_export(request, xform, export_type, options=options) return Response(data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json")
def data(self, request, format='json', **kwargs): """Retrieve the data from the xform using this dataview""" start = request.GET.get("start") limit = request.GET.get("limit") count = request.GET.get("count") sort = request.GET.get("sort") query = request.GET.get("query") export_type = self.kwargs.get('format', request.GET.get("format")) self.object = self.get_object() if export_type is None or export_type in ['json', 'debug']: data = DataView.query_data(self.object, start, limit, str_to_bool(count), sort=sort, filter_query=query) if 'error' in data: raise ParseError(data.get('error')) serializer = self.get_serializer(data, many=True) return Response(serializer.data) else: return custom_response_handler(request, self.object.xform, query, export_type, dataview=self.object)
def forms(self, request, **kwargs): """Add a form to a project or list forms for the project. The request key `xls_file` holds the XLSForm file object. """ project = self.object = self.get_object() if request.method.upper() == 'POST': survey = utils.publish_project_xform(request, project) if isinstance(survey, XForm): if 'formid' in request.data: serializer_cls = XFormSerializer else: serializer_cls = XFormCreateSerializer serializer = serializer_cls(survey, context={'request': request}) published_by_formbuilder = request.data.get( 'published_by_formbuilder' ) if str_to_bool(published_by_formbuilder): MetaData.published_by_formbuilder(survey, 'True') return Response(serializer.data, status=status.HTTP_201_CREATED) return Response(survey, status=status.HTTP_400_BAD_REQUEST) xforms = XForm.objects.filter(project=project) serializer = XFormSerializer(xforms, context={'request': request}, many=True) return Response(serializer.data)
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export(export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta'), request.GET.get('data_id')) elif export_type == Export.OSM_EXPORT: export = generate_osm_export(export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) elif export_type == Export.KML_EXPORT: export = generate_kml_export(export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) else: remove_group_name = False if "remove_group_name" in request.QUERY_PARAMS: remove_group_name = \ str_to_bool(request.QUERY_PARAMS["remove_group_name"]) export = generate_export(export_type, extension, xform.user.username, xform.id_string, None, query, remove_group_name=remove_group_name) audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export( export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta'), request.GET.get('data_id') ) elif export_type == Export.OSM_EXPORT: export = generate_osm_export( export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) elif export_type == Export.KML_EXPORT: export = generate_kml_export( export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) else: remove_group_name = False if "remove_group_name" in request.QUERY_PARAMS: remove_group_name = \ str_to_bool(request.QUERY_PARAMS["remove_group_name"]) export = generate_export( export_type, extension, xform.user.username, xform.id_string, None, query, remove_group_name=remove_group_name ) audit = { "xform": xform.id_string, "export_type": export_type } log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def create_export(request, username, id_string, export_type): owner = get_object_or_404(User, username__iexact=username) xform = get_form({'user': owner, 'id_string__iexact': id_string}) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): return HttpResponseForbidden(_(u'No XLS Template set.')) credential = None if export_type == Export.GOOGLE_SHEETS_EXPORT: credential = _get_google_credential(request) if isinstance(credential, HttpResponseRedirect): return credential query = request.POST.get("query") force_xlsx = request.POST.get('xls') != 'true' # export options group_delimiter = request.POST.get("options[group_delimiter]", '/') if group_delimiter not in ['.', '/']: return HttpResponseBadRequest( _("%s is not a valid delimiter" % group_delimiter)) # default is True, so when dont_.. is yes # split_select_multiples becomes False split_select_multiples = request.POST.get( "options[dont_split_select_multiples]", "no") == "no" binary_select_multiples = getattr(settings, 'BINARY_SELECT_MULTIPLES', False) remove_group_name = request.POST.get("options[remove_group_name]", "false") # external export option meta = request.POST.get("meta") options = { 'group_delimiter': group_delimiter, 'split_select_multiples': split_select_multiples, 'binary_select_multiples': binary_select_multiples, 'remove_group_name': str_to_bool(remove_group_name), 'meta': meta.replace(",", "") if meta else None, 'google_credentials': credential } try: create_async_export(xform, export_type, query, force_xlsx, options) except Export.ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) else: audit = {"xform": xform.id_string, "export_type": export_type} audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'export_type': export_type.upper(), 'id_string': xform.id_string, }, audit, request) return HttpResponseRedirect( reverse(export_list, kwargs={ "username": username, "id_string": id_string, "export_type": export_type }))
def test_str_to_bool(self): self.assertTrue(str_to_bool(True)) self.assertTrue(str_to_bool('True')) self.assertTrue(str_to_bool('TRUE')) self.assertTrue(str_to_bool('true')) self.assertTrue(str_to_bool('t')) self.assertTrue(str_to_bool('1')) self.assertTrue(str_to_bool(1)) self.assertFalse(str_to_bool(False)) self.assertFalse(str_to_bool('False')) self.assertFalse(str_to_bool('F')) self.assertFalse(str_to_bool('random')) self.assertFalse(str_to_bool(234)) self.assertFalse(str_to_bool(0)) self.assertFalse(str_to_bool('0'))
def create_export(request, username, id_string, export_type): """ Create async export tasks view. """ owner = get_object_or_404(User, username__iexact=username) xform = get_form({'user': owner, 'id_string__iexact': id_string}) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): return HttpResponseForbidden(_(u'No XLS Template set.')) credential = None if export_type == Export.GOOGLE_SHEETS_EXPORT: credential = _get_google_credential(request) if isinstance(credential, HttpResponseRedirect): return credential query = request.POST.get("query") force_xlsx = request.POST.get('xls') != 'true' # export options group_delimiter = request.POST.get("options[group_delimiter]", '/') if group_delimiter not in ['.', '/']: return HttpResponseBadRequest( _("%s is not a valid delimiter" % group_delimiter)) # default is True, so when dont_.. is yes # split_select_multiples becomes False split_select_multiples = request.POST.get( "options[dont_split_select_multiples]", "no") == "no" binary_select_multiples = getattr(settings, 'BINARY_SELECT_MULTIPLES', False) remove_group_name = request.POST.get("options[remove_group_name]", "false") value_select_multiples = request.POST.get( "options[value_select_multiples]", "false") # external export option meta = request.POST.get("meta") options = { 'group_delimiter': group_delimiter, 'split_select_multiples': split_select_multiples, 'binary_select_multiples': binary_select_multiples, 'value_select_multiples': str_to_bool(value_select_multiples), 'remove_group_name': str_to_bool(remove_group_name), 'meta': meta.replace(",", "") if meta else None, 'google_credentials': credential } try: create_async_export(xform, export_type, query, force_xlsx, options) except ExportTypeError: return HttpResponseBadRequest( _("%s is not a valid export type" % export_type)) else: audit = {"xform": xform.id_string, "export_type": export_type} audit_log(Actions.EXPORT_CREATED, request.user, owner, _("Created %(export_type)s export on '%(id_string)s'.") % { 'export_type': export_type.upper(), 'id_string': xform.id_string, }, audit, request) return HttpResponseRedirect( reverse( export_list, kwargs={ "username": username, "id_string": id_string, "export_type": export_type }))
def create_async_export(xform, export_type, query, force_xlsx, options=None): username = xform.user.username id_string = xform.id_string @transaction.commit_on_success def _create_export(xform, export_type): return Export.objects.create(xform=xform, export_type=export_type) export = _create_export(xform, export_type) result = None arguments = { 'username': username, 'id_string': id_string, 'export_id': export.id, 'query': query, } if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT, Export.CSV_EXPORT, Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: if options and "group_delimiter" in options: arguments["group_delimiter"] = options["group_delimiter"] if options and "split_select_multiples" in options: arguments["split_select_multiples"] =\ options["split_select_multiples"] if options and "binary_select_multiples" in options: arguments["binary_select_multiples"] =\ options["binary_select_multiples"] if options and "remove_group_name" in options: arguments["remove_group_name"] = \ str_to_bool(options["remove_group_name"]) # start async export if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT]: result = create_xls_export.apply_async((), arguments) elif export_type == Export.CSV_EXPORT: result = create_csv_export.apply_async( (), arguments) elif export_type == Export.CSV_ZIP_EXPORT: result = create_csv_zip_export.apply_async( (), arguments) elif export_type == Export.SAV_ZIP_EXPORT: result = create_sav_zip_export.apply_async( (), arguments) else: raise Export.ExportTypeError elif export_type == Export.ZIP_EXPORT: # start async export result = create_zip_export.apply_async( (), arguments) elif export_type == Export.KML_EXPORT: # start async export result = create_kml_export.apply_async( (), arguments) elif export_type == Export.OSM_EXPORT: # start async export result = create_osm_export.apply_async( (), arguments) elif export_type == Export.EXTERNAL_EXPORT: if options and "token" in options: arguments["token"] = options["token"] if options and "meta" in options: arguments["meta"] = options["meta"] if options and "data_id" in options: arguments["data_id"] = options["data_id"] result = create_external_export.apply_async( (), arguments) else: raise Export.ExportTypeError if result: # when celery is running eager, the export has been generated by the # time we get here so lets retrieve the export object a fresh before we # save if settings.CELERY_ALWAYS_EAGER: export = Export.objects.get(id=export.id) export.task_id = result.task_id export.save() return export, result return None