def set_object_list_and_total_count(self, query, fields, sort, start, limit, is_public_request): try: if not is_public_request: xform = self.get_object() where, where_params = get_where_clause(query) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( self.get_object(), self.request.user, self.object_list) self.object_list = \ self.object_list.order_by('pk')[start: limit] self.total_count = self.object_list.count() elif (sort or limit or start or fields) and not is_public_request: try: query = \ filter_queryset_xform_meta_perms_sql(self.get_object(), self.request.user, query) self.object_list = query_data(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) self.total_count = query_data(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields, count=True)[0].get('count') except NoRecordsPermission: self.object_list = [] self.total_count = 0 else: self.total_count = self.object_list.count() if self.total_count and isinstance(self.object_list, QuerySet): self.etag_hash = get_etag_hash_from_query(self.object_list) elif self.total_count: sql, params, records = get_sql_with_params(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) self.etag_hash = get_etag_hash_from_query(records, sql, params) except ValueError as e: raise ParseError(unicode(e)) except DataError as e: raise ParseError(unicode(e))
def set_object_list(self, query, fields, sort, start, limit, is_public_request): try: enable_etag = True if not is_public_request: xform = self.get_object() self.data_count = xform.num_of_submissions enable_etag = self.data_count <\ SUBMISSION_RETRIEVAL_THRESHOLD where, where_params = get_where_clause(query) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( self.get_object(), self.request.user, self.object_list) self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: query = \ filter_queryset_xform_meta_perms_sql(self.get_object(), self.request.user, query) self.object_list = query_data( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields, json_only=not self.kwargs.get('format') == 'xml') except NoRecordsPermission: self.object_list = [] # ETags are Disabled for XForms with Submissions that surpass # the configured SUBMISSION_RETRIEVAL_THRESHOLD setting if enable_etag: if isinstance(self.object_list, QuerySet): self.etag_hash = get_etag_hash_from_query(self.object_list) else: sql, params, records = get_sql_with_params( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) self.etag_hash = get_etag_hash_from_query( records, sql, params) except ValueError as e: raise ParseError(text(e)) except DataError as e: raise ParseError(text(e))
def test_filter_queryset_xform_meta_perms_sql(self, check_meta_mock): self._publish_transportation_form() query = '{"_id": 1}' result = filter_queryset_xform_meta_perms_sql(self.xform, self.user, query) self.assertEqual(result, query) check_meta_mock.return_value = True alice = self._create_user('alice', 'alice') # no records with self.assertRaises(NoRecordsPermission): filter_queryset_xform_meta_perms_sql(self.xform, alice, query) DataEntryMinorRole.add(alice, self.xform) # meta perms test result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query) self.assertEqual(result, '{"_submitted_by": "alice", "_id": 1}') query = '[{"_id": 1}]' result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query) self.assertEqual(result, '{"_submitted_by": "alice", "_id": 1}') result = filter_queryset_xform_meta_perms_sql(self.xform, alice, None) self.assertEqual(result, '{"_submitted_by": "alice"}')
def test_filter_queryset_xform_meta_perms_sql(self, check_meta_mock): """ Test filter query by meta permissions. """ self._publish_transportation_form() query = '{"_id": 1}' result = filter_queryset_xform_meta_perms_sql(self.xform, self.user, query) self.assertEqual(result, query) check_meta_mock.return_value = True alice = self._create_user('alice', 'alice') # no records with self.assertRaises(NoRecordsPermission): filter_queryset_xform_meta_perms_sql(self.xform, alice, query) DataEntryMinorRole.add(alice, self.xform) # meta perms test result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query) self.assertEqual(result, {"_submitted_by": "alice", "_id": 1}) query = '[{"_id": 1}]' result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query) self.assertEqual(result, {"_submitted_by": "alice", "_id": 1}) result = filter_queryset_xform_meta_perms_sql(self.xform, alice, None) self.assertEqual(result, {"_submitted_by": "alice"})
def set_object_list( self, query, fields, sort, start, limit, is_public_request): try: if not is_public_request: xform = self.get_object() where, where_params = get_where_clause(query) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( self.get_object(), self.request.user, self.object_list) self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: query = \ filter_queryset_xform_meta_perms_sql(self.get_object(), self.request.user, query) self.object_list = query_data(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) except NoRecordsPermission: self.object_list = [] if isinstance(self.object_list, QuerySet): self.etag_hash = get_etag_hash_from_query(self.object_list) else: sql, params, records = get_sql_with_params( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields ) self.etag_hash = get_etag_hash_from_query(records, sql, params) except ValueError as e: raise ParseError(text(e)) except DataError as e: raise ParseError(text(e))
def custom_response_handler(request, xform, query, export_type, token=None, meta=None, dataview=False, filename=None): """ Returns a HTTP response with export file for download. """ export_type = _get_export_type(export_type) if export_type in EXTERNAL_EXPORT_TYPES and \ (token is not None) or (meta is not None): export_type = Export.EXTERNAL_EXPORT options = parse_request_export_options(request.query_params) dataview_pk = hasattr(dataview, 'pk') and dataview.pk options["dataview_pk"] = dataview_pk if dataview: columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) if columns_with_hxl: options['include_hxl'] = include_hxl_row(dataview.columns, list(columns_with_hxl)) try: query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: return Response(data=json.dumps( {"details": _("You don't have permission")}), status=status.HTTP_403_FORBIDDEN, content_type="application/json") if query: options['query'] = query remove_group_name = options.get("remove_group_name") export_id = request.query_params.get("export_id") if export_id: export = get_object_or_404(Export, id=export_id, xform=xform) else: if export_type == Export.GOOGLE_SHEETS_EXPORT: return Response(data=json.dumps( {"details": _("Sheets export only supported in async mode")}), status=status.HTTP_403_FORBIDDEN, content_type="application/json") # check if we need to re-generate, # we always re-generate if a filter is specified def _new_export(): return _generate_new_export(request, xform, query, export_type, dataview_pk=dataview_pk) if should_create_new_export(xform, export_type, options, request=request): export = _new_export() else: export = newest_export_for(xform, export_type, options) if not export.filename and not export.error_message: export = _new_export() log_export(request, xform, export_type) if export_type == Export.EXTERNAL_EXPORT: return external_export_response(export) if export.filename is None and export.error_message: raise exceptions.ParseError(export.error_message) # get extension from file_path, exporter could modify to # xlsx if it exceeds limits _path, ext = os.path.splitext(export.filename) ext = ext[1:] show_date = True if filename is None and export.status == Export.SUCCESSFUL: filename = _generate_filename(request, xform, remove_group_name, dataview_pk=dataview_pk) else: show_date = False response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext], filename, extension=ext, show_date=show_date, file_path=export.filepath) return response
def process_async_export(request, xform, export_type, options=None): """ Check if should generate export or just return the latest export. Rules for regenerating an export are: 1. Filter included on the exports. 2. New submission done. 3. Always regenerate external exports. (External exports uses templates and the template might have changed) :param request: :param xform: :param export_type: :param options: additional export params that may include query: export filter token: template url for xls external reports meta: metadataid that contains the external xls report template url remove_group_name: Flag to determine if group names should appear :return: response dictionary """ # maintain the order of keys while processing the export export_type = _get_export_type(export_type) token = options.get("token") meta = options.get("meta") query = options.get("query") try: query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: payload = {"details": _("You don't have permission")} return Response(data=json.dumps(payload), status=status.HTTP_403_FORBIDDEN, content_type="application/json") else: if query: options['query'] = query if export_type in EXTERNAL_EXPORT_TYPES and \ (token is not None) or (meta is not None): export_type = Export.EXTERNAL_EXPORT if export_type == Export.GOOGLE_SHEETS_EXPORT: credential = _get_google_credential(request) if isinstance(credential, HttpResponseRedirect): return credential options['google_credentials'] = credential.to_json() if should_create_new_export(xform, export_type, options, request=request)\ or export_type == Export.EXTERNAL_EXPORT: resp = { u'job_uuid': _create_export_async(xform, export_type, query, False, options=options) } else: print('Do not create a new export.') export = newest_export_for(xform, export_type, options) if not export.filename: # tends to happen when using newest_export_for. resp = { u'job_uuid': _create_export_async(xform, export_type, query, False, options=options) } else: resp = export_async_export_response(request, export) return resp