def set_object_list_and_total_count(self, query, fields, sort, start,
limit, is_public_request):
try:
if not is_public_request:
xform = self.get_object()
where, where_params = get_where_clause(query)
if where:
self.object_list = self.object_list.extra(where=where,
params=where_params)
if (start and limit or limit) and (not sort and not fields):
start = start if start is not None else 0
limit = limit if start is None or start == 0 else start + limit
self.object_list = filter_queryset_xform_meta_perms(
self.get_object(), self.request.user, self.object_list)
self.object_list = \
self.object_list.order_by('pk')[start: limit]
self.total_count = self.object_list.count()
elif (sort or limit or start or fields) and not is_public_request:
try:
query = \
filter_queryset_xform_meta_perms_sql(self.get_object(),
self.request.user,
query)
self.object_list = query_data(xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields)
self.total_count = query_data(xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields,
count=True)[0].get('count')
except NoRecordsPermission:
self.object_list = []
self.total_count = 0
else:
self.total_count = self.object_list.count()
if self.total_count and isinstance(self.object_list, QuerySet):
self.etag_hash = get_etag_hash_from_query(self.object_list)
elif self.total_count:
sql, params, records = get_sql_with_params(xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields)
self.etag_hash = get_etag_hash_from_query(records, sql, params)
except ValueError as e:
raise ParseError(unicode(e))
except DataError as e:
raise ParseError(unicode(e))
def set_object_list(self, query, fields, sort, start, limit,
is_public_request):
try:
enable_etag = True
if not is_public_request:
xform = self.get_object()
self.data_count = xform.num_of_submissions
enable_etag = self.data_count <\
SUBMISSION_RETRIEVAL_THRESHOLD
where, where_params = get_where_clause(query)
if where:
self.object_list = self.object_list.extra(where=where,
params=where_params)
if (start and limit or limit) and (not sort and not fields):
start = start if start is not None else 0
limit = limit if start is None or start == 0 else start + limit
self.object_list = filter_queryset_xform_meta_perms(
self.get_object(), self.request.user, self.object_list)
self.object_list = self.object_list[start:limit]
elif (sort or limit or start or fields) and not is_public_request:
try:
query = \
filter_queryset_xform_meta_perms_sql(self.get_object(),
self.request.user,
query)
self.object_list = query_data(
xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields,
json_only=not self.kwargs.get('format') == 'xml')
except NoRecordsPermission:
self.object_list = []
# ETags are Disabled for XForms with Submissions that surpass
# the configured SUBMISSION_RETRIEVAL_THRESHOLD setting
if enable_etag:
if isinstance(self.object_list, QuerySet):
self.etag_hash = get_etag_hash_from_query(self.object_list)
else:
sql, params, records = get_sql_with_params(
xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields)
self.etag_hash = get_etag_hash_from_query(
records, sql, params)
except ValueError as e:
raise ParseError(text(e))
except DataError as e:
raise ParseError(text(e))
def test_filter_queryset_xform_meta_perms_sql(self, check_meta_mock):
self._publish_transportation_form()
query = '{"_id": 1}'
result = filter_queryset_xform_meta_perms_sql(self.xform, self.user,
query)
self.assertEqual(result, query)
check_meta_mock.return_value = True
alice = self._create_user('alice', 'alice')
# no records
with self.assertRaises(NoRecordsPermission):
filter_queryset_xform_meta_perms_sql(self.xform, alice, query)
DataEntryMinorRole.add(alice, self.xform)
# meta perms test
result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query)
self.assertEqual(result, '{"_submitted_by": "alice", "_id": 1}')
query = '[{"_id": 1}]'
result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query)
self.assertEqual(result, '{"_submitted_by": "alice", "_id": 1}')
result = filter_queryset_xform_meta_perms_sql(self.xform, alice, None)
self.assertEqual(result, '{"_submitted_by": "alice"}')
def test_filter_queryset_xform_meta_perms_sql(self, check_meta_mock):
"""
Test filter query by meta permissions.
"""
self._publish_transportation_form()
query = '{"_id": 1}'
result = filter_queryset_xform_meta_perms_sql(self.xform, self.user,
query)
self.assertEqual(result, query)
check_meta_mock.return_value = True
alice = self._create_user('alice', 'alice')
# no records
with self.assertRaises(NoRecordsPermission):
filter_queryset_xform_meta_perms_sql(self.xform, alice, query)
DataEntryMinorRole.add(alice, self.xform)
# meta perms test
result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query)
self.assertEqual(result, {"_submitted_by": "alice", "_id": 1})
query = '[{"_id": 1}]'
result = filter_queryset_xform_meta_perms_sql(self.xform, alice, query)
self.assertEqual(result, {"_submitted_by": "alice", "_id": 1})
result = filter_queryset_xform_meta_perms_sql(self.xform, alice, None)
self.assertEqual(result, {"_submitted_by": "alice"})
def set_object_list(
self, query, fields, sort, start, limit, is_public_request):
try:
if not is_public_request:
xform = self.get_object()
where, where_params = get_where_clause(query)
if where:
self.object_list = self.object_list.extra(where=where,
params=where_params)
if (start and limit or limit) and (not sort and not fields):
start = start if start is not None else 0
limit = limit if start is None or start == 0 else start + limit
self.object_list = filter_queryset_xform_meta_perms(
self.get_object(), self.request.user, self.object_list)
self.object_list = self.object_list[start:limit]
elif (sort or limit or start or fields) and not is_public_request:
try:
query = \
filter_queryset_xform_meta_perms_sql(self.get_object(),
self.request.user,
query)
self.object_list = query_data(xform, query=query,
sort=sort, start_index=start,
limit=limit, fields=fields)
except NoRecordsPermission:
self.object_list = []
if isinstance(self.object_list, QuerySet):
self.etag_hash = get_etag_hash_from_query(self.object_list)
else:
sql, params, records = get_sql_with_params(
xform, query=query, sort=sort, start_index=start,
limit=limit, fields=fields
)
self.etag_hash = get_etag_hash_from_query(records, sql, params)
except ValueError as e:
raise ParseError(text(e))
except DataError as e:
raise ParseError(text(e))
def custom_response_handler(request,
xform,
query,
export_type,
token=None,
meta=None,
dataview=False,
filename=None):
"""
Returns a HTTP response with export file for download.
"""
export_type = _get_export_type(export_type)
if export_type in EXTERNAL_EXPORT_TYPES and \
(token is not None) or (meta is not None):
export_type = Export.EXTERNAL_EXPORT
options = parse_request_export_options(request.query_params)
dataview_pk = hasattr(dataview, 'pk') and dataview.pk
options["dataview_pk"] = dataview_pk
if dataview:
columns_with_hxl = get_columns_with_hxl(xform.survey.get('children'))
if columns_with_hxl:
options['include_hxl'] = include_hxl_row(dataview.columns,
list(columns_with_hxl))
try:
query = filter_queryset_xform_meta_perms_sql(xform, request.user,
query)
except NoRecordsPermission:
return Response(data=json.dumps(
{"details": _("You don't have permission")}),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
if query:
options['query'] = query
remove_group_name = options.get("remove_group_name")
export_id = request.query_params.get("export_id")
if export_id:
export = get_object_or_404(Export, id=export_id, xform=xform)
else:
if export_type == Export.GOOGLE_SHEETS_EXPORT:
return Response(data=json.dumps(
{"details": _("Sheets export only supported in async mode")}),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
# check if we need to re-generate,
# we always re-generate if a filter is specified
def _new_export():
return _generate_new_export(request,
xform,
query,
export_type,
dataview_pk=dataview_pk)
if should_create_new_export(xform,
export_type,
options,
request=request):
export = _new_export()
else:
export = newest_export_for(xform, export_type, options)
if not export.filename and not export.error_message:
export = _new_export()
log_export(request, xform, export_type)
if export_type == Export.EXTERNAL_EXPORT:
return external_export_response(export)
if export.filename is None and export.error_message:
raise exceptions.ParseError(export.error_message)
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
_path, ext = os.path.splitext(export.filename)
ext = ext[1:]
show_date = True
if filename is None and export.status == Export.SUCCESSFUL:
filename = _generate_filename(request,
xform,
remove_group_name,
dataview_pk=dataview_pk)
else:
show_date = False
response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext],
filename,
extension=ext,
show_date=show_date,
file_path=export.filepath)
return response
def process_async_export(request, xform, export_type, options=None):
"""
Check if should generate export or just return the latest export.
Rules for regenerating an export are:
1. Filter included on the exports.
2. New submission done.
3. Always regenerate external exports.
(External exports uses templates and the template might have
changed)
:param request:
:param xform:
:param export_type:
:param options: additional export params that may include
query: export filter
token: template url for xls external reports
meta: metadataid that contains the external xls report template url
remove_group_name: Flag to determine if group names should appear
:return: response dictionary
"""
# maintain the order of keys while processing the export
export_type = _get_export_type(export_type)
token = options.get("token")
meta = options.get("meta")
query = options.get("query")
try:
query = filter_queryset_xform_meta_perms_sql(xform, request.user,
query)
except NoRecordsPermission:
payload = {"details": _("You don't have permission")}
return Response(data=json.dumps(payload),
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
else:
if query:
options['query'] = query
if export_type in EXTERNAL_EXPORT_TYPES and \
(token is not None) or (meta is not None):
export_type = Export.EXTERNAL_EXPORT
if export_type == Export.GOOGLE_SHEETS_EXPORT:
credential = _get_google_credential(request)
if isinstance(credential, HttpResponseRedirect):
return credential
options['google_credentials'] = credential.to_json()
if should_create_new_export(xform, export_type, options, request=request)\
or export_type == Export.EXTERNAL_EXPORT:
resp = {
u'job_uuid':
_create_export_async(xform,
export_type,
query,
False,
options=options)
}
else:
print('Do not create a new export.')
export = newest_export_for(xform, export_type, options)
if not export.filename:
# tends to happen when using newest_export_for.
resp = {
u'job_uuid':
_create_export_async(xform,
export_type,
query,
False,
options=options)
}
else:
resp = export_async_export_response(request, export)
return resp
