def retrieve(self, request, pk=None, **kwargs):
"""
Download the JSON release file using the ReleaseInfo object_id
Example: http://127.0.0.1:8000/api/release-download/18bc23da-cdbf-420e-a6aa-3d9ecdb10c20/
"""
release_info = get_object_or_404(ReleaseInfo, object_id=pk)
if not release_info.dp_release_json_file:
user_msg = 'The Release does not include a JSON file.'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# get an open file handle
try:
file_handle = release_info.dp_release_json_file.open()
except ValueError as err_obj:
user_msg = f'Not able to read the JSON Release file. (1) ({err_obj})'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
except Exception as err_obj:
user_msg = f'Not able to read the JSON Release file. (2) ({err_obj})'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# send file
response = FileResponse(file_handle, content_type='application/json')
response['Content-Length'] = release_info.dp_release_json_file.size
download_fname = f'release_{release_info.object_id}.json'
response[
'Content-Disposition'] = f'attachment; filename="{download_fname}"'
return response
def pdf(self, request, pk=None):
"""
Download the PDF release file using the ReleaseInfo object_id
Note: URL linked to code in opendp_apps/analysis/models.py -> download_pdf_url()
Example: http://127.0.0.1:8000/api/release-download/18bc23da-cdbf-420e-a6aa-3d9ecdb10c20/pdf/
"""
release_info = get_object_or_404(ReleaseInfo, object_id=pk)
if not release_info.dp_release_pdf_file:
user_msg = 'The Release does not include a PDF file.'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# get an open file handle
try:
file_handle = release_info.dp_release_pdf_file.open()
except ValueError as err_obj:
user_msg = f'Not able to read the PDF Release file. (1) ({err_obj})'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
except Exception as err_obj:
user_msg = f'Not able to read the PDF Release file. (2) ({err_obj})'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# send file
response = FileResponse(file_handle, content_type='application/pdf')
response['Content-Length'] = release_info.dp_release_pdf_file.size
download_fname = f'release_{release_info.object_id}.pdf'
response[
'Content-Disposition'] = f'attachment; filename="{download_fname}"'
return response
def list(self, request):
"""Only used so that these endpoints show up in the docs"""
user_msg = ('ProfilingViewSet: custom actions only.'
' In API docs, see "Extra Actions"')
return Response(get_json_error(user_msg),
status=status.HTTP_501_NOT_IMPLEMENTED)
return Response(get_json_success())
def retrieve_profile(self, request, *args, **kwargs):
"""Retrieve the DataSetInfo.profile_variables in JSON format.
- Input: DataSetInfo.object_id (UUID in string format)
- Output: DataSetInfo.profile_variables in JSON format
NOTES:
- The logged in user must match the DataSetInfo.creator
"""
print('\n>>request', request)
print('\n>>request.data', request.data, type(request.data))
# Is this a object_id a valid UUID?
#
ois = DatasetObjectIdSerializer(data=request.data)
if not ois.is_valid():
print(ois.errors)
if 'object_id' in ois.errors:
user_msg = '"object_id" error: %s' % (
ois.errors['object_id'][0])
else:
user_msg = 'Not a valid "object_id"'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# Is there a related DataSetInfo where the logged in user is the
# DataSetInfo creator?
#
dsi_info = ois.get_dataset_info_with_user_check(request.user)
if not dsi_info.success:
return Response(get_json_error(dsi_info.message),
status=status.HTTP_404_NOT_FOUND)
# Does the profile exist?
#
dsi_object = dsi_info.data
if not dsi_object.profile_variables:
user_msg = 'Dataset not profiled'
# status for profile?
return Response(get_json_error(user_msg))
# Profile found!
#
return Response(
get_json_success('Profile found',
data=dsi_object.get_profile_variables()))
def run_direct_profile(self, request, *args, **kwargs):
"""TEST ONLY - Profile a DataSetInfo object, returning the DataSetInfo.profile_variables in JSON format.
- Input: DataSetInfo.object_id (UUID in string format)
- Output: DataSetInfo.profile_variables in JSON format
NOTES:
- TEST ONLY: Uses the main web server thread
- The logged in user must match the DataSetInfo.creator
- The response is returned asynchronously, via a websocket
- If the profile already exists, it will be returned asynchronously.
- If the DataSetInfo object is a DataverseFileInfo object, if necessary, this endpoint will both download the dataset in order to profile it.
"""
# Is this a object_id a valid UUID?
#
ois = DatasetObjectIdSerializer(data=request.data)
if not ois.is_valid():
#print(ois.errors)
if 'object_id' in ois.errors:
user_msg = '"object_id" error: %s' % (
ois.errors['object_id'][0])
else:
user_msg = 'Not a valid "object_id"'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# Is there a related DataSetInfo where the logged in user is the
# DataSetInfo creator?
#
dsi_info = ois.get_dataset_info_with_user_check(request.user)
if not dsi_info.success:
return Response(get_json_error(dsi_info.message),
status=status.HTTP_404_NOT_FOUND)
websocket_id = get_websocket_id(request)
ddi_info = profile_dataset_info(ois.get_object_id(),
websocket_id=websocket_id)
if not ddi_info.success:
return Response(get_json_error(ddi_info.message))
dp_util = ddi_info.data
user_msg = ('Profiling complete.')
return Response(
get_json_success(user_msg, data=dp_util.get_profile_variables()))
def create(self, request, *args, **kwargs):
"""
Create an AnalysisPlan object with default values
"""
# Is this a object_id a valid UUID?
#
ois = DatasetObjectIdSerializer(data=request.data)
if not ois.is_valid():
#print(ois.errors)
if 'object_id' in ois.errors:
user_msg = '"object_id" error: %s' % (
ois.errors['object_id'][0])
else:
user_msg = 'Not a valid "object_id"'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# Is there a related DataSetInfo where the logged in user is the
# DataSetInfo creator?
#
dsi_info = ois.get_dataset_info_with_user_check(request.user)
if not dsi_info.success:
return Response(get_json_error(dsi_info.message),
status=status.HTTP_404_NOT_FOUND)
# Use the AnalysisPlanUtil to create an AnalysisPlan
# with default values
#
plan_util = AnalysisPlanUtil.create_plan(ois.get_object_id(),
request.user)
# Did AnalysisPlan creation work?
if plan_util.success:
# Yes, it worked!
new_plan = plan_util.data # "data" holds the AnalysisPlan object
serializer = AnalysisPlanSerializer(new_plan) # serialize the data
# Return it
return Response(serializer.data, status=status.HTTP_201_CREATED)
# Nope! Error encountered!
return Response(get_json_error(plan_util.message),
status=plan_util.data)
def manifest_test_params_view(request, object_id):
"""
Return the ManifestTestParams in JSON format.
(Limit this to superusers)
"""
mparams = ManifestTestParams.objects.filter(object_id=object_id).first()
if not mparams:
return JsonResponse(get_json_error('Object not found'),
status=HTTPStatus.NOT_FOUND)
return JsonResponse(get_json_success('Success', data=mparams.as_dict()))
def partial_update(self, request, *args, **kwargs):
"""Make updates to the AnalysisPlan object"""
# print('>>> partial_update: ', request.data)
acceptable_fields = ['variable_info', 'dp_statistics', 'user_step']
problem_fields = []
fields_to_update = []
for field in request.data.keys():
if field not in acceptable_fields:
problem_fields.append(field)
else:
fields_to_update.append(field)
if problem_fields:
problem_field_list = ', '.join([str(f) for f in problem_fields])
user_msg = f'{astatic.ERR_MSG_FIELDS_NOT_UPDATEABLE}: {problem_field_list}'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
if not fields_to_update:
return Response(get_json_error(f'There are no fields to update'),
status=status.HTTP_400_BAD_REQUEST)
return super(AnalysisPlanViewSet,
self).partial_update(request, *args, **kwargs)
def list(self, request, *args, **kwargs):
"""Retrieve all active RegisteredDataverse objects"""
serializer = RegisteredDataverseSerializer(self.queryset, many=True)
num_dvs = len(serializer.data)
success = False
message = "No registered Dataverses found."
if num_dvs > 0:
success = True
message = f"{num_dvs} registered Dataverse(s) found."
if success:
return Response(get_json_success(message,
data=dict(
count=num_dvs,
dataverses=serializer.data)),
status=status.HTTP_200_OK)
return Response(get_json_error(message),
status=status.HTTP_404_NOT_FOUND)
def create(self, request, *args, **kwargs):
"""Expects JSON. Given object_ids for OpenDPUser and DataverseHandoff objects,
retrieve the user's information from Dataverse and create a DataverseUser"""
# ----------------------------------
# Validate the input
# ----------------------------------
# print(f"data: {request.data}")
request_data = request.data.copy()
user_id = request.data.get('user')
handoff_id = request.data.get('dv_handoff')
request_data['handoff'] = handoff_id
request_data['user'] = user_id
handoff_obj = get_object_or_error_response(DataverseHandoff,
object_id=handoff_id)
try:
dataverse_user = DataverseUser.objects.get(
user__object_id=user_id,
dv_installation=handoff_obj.dv_installation)
opendp_user = dataverse_user.user
except DataverseUser.DoesNotExist:
# ----------------------------------
# Create the DataverseUser object
# ----------------------------------
dataverse_user_serializer = DataverseUserSerializer(
data=request_data, context={'request': request})
if not dataverse_user_serializer.is_valid():
# print("INVALID SERIALIZER")
return Response(dataverse_user_serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
try:
dataverse_user = dataverse_user_serializer.save()
except DataverseHandoff.DoesNotExist:
return Response(dataverse_user_serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
except DataverseUser.DoesNotExist:
return Response(dataverse_user_serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
opendp_user = dataverse_user_serializer.validated_data.get('user')
# ----------------------------------
# Call the Dataverse API
# ----------------------------------
site_url = handoff_obj.dv_installation.dataverse_url
# print('-- site_url', site_url)
api_general_token = dataverse_user.dv_general_token
dataverse_client = DataverseClient(site_url, api_general_token)
try:
dataverse_response = dataverse_client.get_user_info(
user_api_token=api_general_token)
except InvalidSchema:
return Response(
get_json_error(f'The Site {site_url} is not valid'),
status=status.HTTP_400_BAD_REQUEST)
except JSONDecodeError:
return Response(
get_json_error(f'Error reading data from {site_url}'),
status=status.HTTP_400_BAD_REQUEST)
if dataverse_response.success is not True:
return Response(get_json_error(dataverse_response.message),
status=status.HTTP_400_BAD_REQUEST)
try:
handler = DataverseUserHandler(opendp_user.id, site_url,
api_general_token,
dataverse_response.__dict__)
update_response = handler.update_dataverse_user()
except DataverseResponseError as ex:
return Response(get_json_error(f'Error {ex}'),
status=status.HTTP_400_BAD_REQUEST)
return Response(get_json_success(
'success', data={'dv_user': dataverse_user.object_id}),
status=status.HTTP_201_CREATED)
def update(self, request, object_id=None, *args, **kwargs):
"""NOT REALLY USED!!! e.g. create is really create_or_update"""
"""Update the Dataverse User. Expects JSON"""
# ----------------------------------
# Validate the input
# ----------------------------------
print(f"data: {request.data}\tpk: {object_id}")
dataverse_user = get_object_or_error_response(DataverseUser,
object_id=object_id)
opendp_user = dataverse_user.user
request.data['user'] = opendp_user.object_id
dataverse_user_serializer = DataverseUserSerializer(
data=request.data, context={'request': request})
if dataverse_user_serializer.is_valid():
try:
dataverse_user = dataverse_user_serializer.update(
dataverse_user, request.data)
except DataverseHandoff.DoesNotExist:
return JsonResponse(
{
'success': False,
'message': 'No such DataVerse exists'
},
status=status.HTTP_400_BAD_REQUEST)
else:
return Response(dataverse_user_serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
opendp_user = dataverse_user_serializer.validated_data.get('user')
if not opendp_user:
return Response({
'success': False,
'message': 'No OpenDP user found'
})
# ----------------------------------
# Call the Dataverse API
# ----------------------------------
dv_handoff = get_object_or_error_response(
DataverseHandoff, object_id=request.data['dv_handoff'])
site_url = dv_handoff.dv_installation.dataverse_url
api_general_token = dataverse_user.dv_general_token
dataverse_client = DataverseClient(site_url, api_general_token)
try:
dataverse_response = dataverse_client.get_user_info(
user_api_token=api_general_token)
except InvalidSchema:
return JsonResponse(
get_json_error(f'The Site {site_url} is not valid'),
status=400)
except JSONDecodeError:
return JsonResponse(
get_json_error(f'Error reading data from {site_url}'),
status=status.HTTP_400_BAD_REQUEST)
if dataverse_response.success is not True:
return JsonResponse(get_json_error(dataverse_response.message),
status=400)
# ----------------------------------
# Update the DataverseUser object
# ----------------------------------
try:
handler = DataverseUserHandler(opendp_user.id, site_url,
api_general_token,
dataverse_response.__dict__)
update_resp = handler.update_dataverse_user()
if update_resp.success:
updated_dv_user = update_resp.data
updated_dv_user.save()
else:
return JsonResponse(get_json_error(update_resp.message),
status=status.HTTP_400_BAD_REQUEST)
except DataverseResponseError as ex:
return JsonResponse(get_json_error(f'Error {ex}'),
status=status.HTTP_400_BAD_REQUEST)
return JsonResponse(get_json_success(
'updated', data=dict(dv_user=updated_dv_user.object_id)),
status=201)
def create(self, request, *args, **kwargs):
"""
Note: No data is saved. This endpoint is used for validation.
endpoint: /api/validation/
Example POST input:
{
"analysis_plan_id": "616b5167-4ce8-4def-85dc-6f0d8de2316c",
"dp_statistics": [
{
"statistic": "mean",
"variable": "EyeHeight",
"epsilon": 0.6,
"delta": 0,
"error": "",
"missing_values_handling": "insert_fixed",
"handle_as_fixed": false,
"fixed_value": "5.0",
"locked": false,
"label": "EyeHeight"
}
]
}
-- Example outputs --
(1) Overall error
Status code: 400
{
"success": false,
"message": "The depositor setup info has an invalid epsilon value: 4.0"
}
(2) Single statistic error -- even if only 1 statistic submitted
Status code: 200 - NOTE status code is 200!
{
"success": true,
"message": "validation results returned",
"data": [
{
"var_name": "BlinkDuration",
"statistic": "mean",
"valid": false,
"message": "As a rule of thumb, epsilon should not be less than 0.001 nor greater than 1."
}
]
}
(2) Single statistic success -- even if only 1 statistic submitted
Status code: 200 - NOTE status code is 200!
{
"success": true,
"message": "validation results returned",
"data": [
{
"var_name": "EyeHeight",
"statistic": "mean",
"valid": true,
"message": null
}
]
}
(3) Mixed success and error -- even if only 1 statistic submitted
Status code: 200 - NOTE status code is 200!
{
"success": true,
"message": "validation results returned",
"data": [
{
"var_name": "EyeHeight",
"statistic": "mean",
"valid": true,
"message": null
},
{
"var_name": "BlinkDuration",
"statistic": "mean",
"valid": false,
"message": "The running epsilon (1.45) exceeds the max epsilon (1.0)"
}
]
}
"""
#print('>> ReleaseView.create >>>', request.data)
release_info_serializer = ReleaseValidationSerializer(
data=request.data)
if not release_info_serializer.is_valid():
print('release_info_serializer.errors',
release_info_serializer.errors)
return Response(get_json_error(
'Field validation failed',
errors=release_info_serializer.errors),
status=status.HTTP_200_OK)
#status=status.HTTP_400_BAD_REQUEST)
save_result = release_info_serializer.save(**dict(
opendp_user=request.user))
#print(save_result.success)
if not save_result.success:
#print(save_result.message)
return Response(get_json_error(save_result.message),
status=status.HTTP_400_BAD_REQUEST)
#print(save_result.data)
return Response(get_json_success('validation results returned',
data=save_result.data),
status=status.HTTP_200_OK)
def create(self, request, *args, **kwargs):
"""
Override create to create a release based on a saved/pre-validated AnalysisPlan.dp_statistics
endpoint: /api/release/
Example POST input: `{"analysis_plan_id": "616b5167-4ce8-4def-85dc-6f0d8de2316c"}`
-- Example outputs --
(1) Overall error
Status code: 400
{
"success": false,
"message": "The statistic 'EyeHeight' had error xyz!"
}
(2) Release success
Status code: 201
{
"success": true,
"message": "release worked!",
"data":
(see: opendp_apps.release_info_formatter.get_release_data())
"""
# Get the AnalysisPlan object_id
# - Bit redundant in the serializer re: retrieving plan but only using object_id--but okay!
# - e.g. Passing along the object_id only will lead to later running chain
# in a separate process and responding by websocket/lambda, etc.
#
serializer = AnalysisPlanObjectIdSerializer(data=request.data)
if not serializer.is_valid():
print(serializer.errors)
if 'object_id' in serializer.errors:
user_msg = '"object_id" error: %s' % (
serializer.errors['object_id'][0])
else:
user_msg = 'Not a valid AnalysisPlan "object_id"'
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# We have a good object_id!
#
analysis_plan_id = serializer.get_object_id()
# For longer releases, Async this!!
# Async: the validate_util process!
validate_util = ValidateReleaseUtil.compute_mode(
request.user, analysis_plan_id, run_dataverse_deposit=True)
if validate_util.has_error():
# This is a big error, check for it before evaluating individual statistics
#
user_msg = validate_util.get_err_msg()
print('release_view.create(...) user_msg', user_msg)
# Can you return a 400 / raise an Exception here with the error message?
# How should this be used?
return Response(get_json_error(user_msg),
status=status.HTTP_400_BAD_REQUEST)
# It worked! Return the release!
release_info_obj = validate_util.get_new_release_info_object()
serializer = ReleaseInfoSerializer(release_info_obj,
context={'request': request})
return Response(data=serializer.data, status=status.HTTP_201_CREATED)
