def transfer_patch(self):
"""
Attempt to cancel a transfer job.
"""
# Perform token validation. Read data from the process_info file.
try:
destination_token = get_destination_token(self.request)
source_token = get_source_token(self.request)
self.ticket_number = '{}_{}'.format(hash_tokens(source_token),
hash_tokens(destination_token))
process_data = get_process_info_data(self.ticket_number)
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
# Wait until the spawned off process has started to cancel the transfer
while process_data['resource_transfer_in'][
'function_process_id'] is None:
try:
process_data = get_process_info_data(self.ticket_number)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
transfer_process_data = process_data['resource_transfer_in']
# If transfer is still in progress then cancel the subprocess
if transfer_process_data['status'] == 'in_progress':
for process in multiprocessing.active_children():
if process.pid == transfer_process_data['function_process_id']:
process.kill()
process.join()
transfer_process_data['status'] = 'failed'
transfer_process_data[
'message'] = 'Transfer was cancelled by the user'
transfer_process_data['status_code'] = '499'
transfer_process_data['expiration'] = str(timezone.now() +
relativedelta(
hours=1))
update_or_create_process_info(transfer_process_data,
'resource_transfer_in',
self.ticket_number)
return Response(data={
'status_code':
transfer_process_data['status_code'],
'message':
transfer_process_data['message']
},
status=status.HTTP_200_OK)
# If transfer is finished then don't attempt to cancel subprocess
else:
return Response(data={
'status_code': transfer_process_data['status_code'],
'message': transfer_process_data['message']
},
status=status.HTTP_406_NOT_ACCEPTABLE)
def upload_get(self):
"""
Get the status of an upload job.
"""
# Perform token validation. Read data from the process_info file.
try:
destination_token = get_destination_token(self.request)
self.ticket_number = hash_tokens(destination_token)
self.process_data = get_process_info_data(self.ticket_number)
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
try:
upload_process_data = self.process_data['resource_upload']
except KeyError:
return Response(data={
'error':
'PresQT Error: "resource_upload" not found in process_info file.'
},
status=status.HTTP_400_BAD_REQUEST)
upload_status = upload_process_data['status']
total_files = upload_process_data['upload_total_files']
files_finished = upload_process_data['upload_files_finished']
job_percentage = calculate_job_percentage(total_files, files_finished)
data = {
'status_code': upload_process_data['status_code'],
'status': upload_status,
'message': upload_process_data['message'],
'job_percentage': job_percentage
}
if upload_status == 'finished':
http_status = status.HTTP_200_OK
data['status'] = upload_status
data['failed_fixity'] = upload_process_data['failed_fixity']
data['resources_ignored'] = upload_process_data[
'resources_ignored']
data['resources_updated'] = upload_process_data[
'resources_updated']
data['link_to_resource'] = upload_process_data['link_to_resource']
data['job_percentage'] = 99
else:
if upload_status == 'in_progress':
http_status = status.HTTP_202_ACCEPTED
data['job_percentage'] = job_percentage
else:
http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
return Response(status=http_status, data=data)
def get(self, request, ticket_number):
"""
Check in on the resource's transfer process state.
Parameters
----------
ticket_number : str
The ticket number of the transfer being prepared.
Returns
-------
200: OK
"""
# Perform token validation. Read data from the process_info file.
try:
destination_token = get_destination_token(request)
source_token = get_source_token(request)
process_data = get_process_info_data('transfers', ticket_number)
process_token_validation(hash_tokens(destination_token),
process_data, 'presqt-destination-token')
process_token_validation(hash_tokens(source_token), process_data, 'presqt-source-token')
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
transfer_status = process_data['status']
data = {'status_code': process_data['status_code'], 'message': process_data['message']}
if transfer_status == 'finished':
http_status = status.HTTP_200_OK
data['failed_fixity'] = process_data['failed_fixity']
data['resources_ignored'] = process_data['resources_ignored']
data['resources_updated'] = process_data['resources_updated']
else:
if transfer_status == 'in_progress':
http_status = status.HTTP_202_ACCEPTED
else:
http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
return Response(status=http_status, data=data)
def patch(self, request, ticket_number):
"""
Cancel the resource upload process on the server. Update the process_info.json
file appropriately.
Parameters
----------
ticket_number : str
The ticket number of the upload being prepared.
Returns
-------
200: OK
{
"status_code": "499",
"message": "Upload was cancelled by the user"
}
400: Bad Request
{
"error": "'presqt-destination-token' missing in the request headers."
}
401: Unauthorized
{
"error": "Header 'presqt-destination-token' does not match the
'presqt-destination-token' for this server process."
}
404: Not Found
{
"error": "Invalid ticket number, '1234'."
}
406: Not Acceptable
{
"status_code": "200",
"message": "Upload Successful"
}
"""
# Perform token validation. Read data from the process_info file.
try:
token = get_destination_token(request)
data = get_process_info_data('uploads', ticket_number)
process_token_validation(hash_tokens(token), data,
'presqt-destination-token')
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
# Wait until the spawned off process has started to cancel the upload
while data['function_process_id'] is None:
try:
data = get_process_info_data('uploads', ticket_number)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
# If upload is still in progress then cancel the subprocess
if data['status'] == 'in_progress':
for process in multiprocessing.active_children():
if process.pid == data['function_process_id']:
process.kill()
process.join()
data['status'] = 'failed'
data['message'] = 'Upload was cancelled by the user'
data['status_code'] = '499'
data['expiration'] = str(timezone.now() +
relativedelta(hours=1))
process_info_path = 'mediafiles/uploads/{}/process_info.json'.format(
ticket_number)
write_file(process_info_path, data, True)
return Response(data={
'status_code': data['status_code'],
'message': data['message']
},
status=status.HTTP_200_OK)
# If upload is finished then don't attempt to cancel subprocess
else:
return Response(data={
'status_code': data['status_code'],
'message': data['message']
},
status=status.HTTP_406_NOT_ACCEPTABLE)
def get(self, request, ticket_number):
"""
Check in on the resource's upload process state.
Parameters
----------
ticket_number : str
The ticket number of the upload being prepared.
Returns
-------
200: OK
{
"status_code": "200",
"message": "Upload successful",
"failed_fixity": [],
"resources_ignored": [],
"resources_updated": []
}
202: Accepted
{
"status_code": null,
"message": "Upload is being processed on the server"
}
400: Bad Request
{
"error": "PresQT Error: 'presqt-destination-token' missing in the request headers."
}
401: Unauthorized
{
"error": "PresQT Error: Header 'presqt-destination-token' does not match the
'presqt-destination-token' for this server process."
}
404: Not Found
{
"error": "PresQT Error: Invalid ticket number, '1234'."
}
500: Internal Server Error
{
"status_code": "404",
"message": "Resource with id 'bad_id' not found for this user."
}
"""
# Perform token validation. Read data from the process_info file.
try:
token = get_destination_token(request)
process_data = get_process_info_data('uploads', ticket_number)
process_token_validation(hash_tokens(token), process_data,
'presqt-destination-token')
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
upload_status = process_data['status']
data = {
'status_code': process_data['status_code'],
'message': process_data['message']
}
if upload_status == 'finished':
http_status = status.HTTP_200_OK
data['failed_fixity'] = process_data['failed_fixity']
data['resources_ignored'] = process_data['resources_ignored']
data['resources_updated'] = process_data['resources_updated']
else:
if upload_status == 'in_progress':
http_status = status.HTTP_202_ACCEPTED
else:
http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
return Response(status=http_status, data=data)
def transfer_post(self):
"""
Transfer resources to a specific existing target resource or create a new target resource.
Returns
-------
Response object in JSON format
"""
try:
self.destination_token = get_destination_token(self.request)
self.source_token = get_source_token(self.request)
self.email = get_user_email_opt(self.request)
self.file_duplicate_action = file_duplicate_action_validation(
self.request)
self.keyword_action = keyword_action_validation(self.request)
self.fairshare_evaluator_action = fairshare_evaluator_validation(
self.request)
self.source_target_name, self.source_resource_id, self.keywords = transfer_post_body_validation(
self.request)
target_valid, self.infinite_depth = target_validation(
self.destination_target_name, self.action)
target_validation(self.source_target_name, 'resource_transfer_out')
transfer_target_validation(self.source_target_name,
self.destination_target_name)
self.supports_keywords = get_keyword_support(
self.source_target_name, self.destination_target_name)
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
# Generate ticket number
self.ticket_number = '{}_{}'.format(
hash_tokens(self.source_token),
hash_tokens(self.destination_token))
ticket_path = os.path.join("mediafiles", "jobs",
str(self.ticket_number))
self.ticket_path = os.path.join('mediafiles', 'jobs',
str(self.ticket_number), 'transfer')
# Create directory and process_info json file
self.process_info_obj = {
'presqt-source-token': hash_tokens(self.source_token),
'presqt-destination-token': hash_tokens(self.destination_token),
'status': 'in_progress',
'expiration': str(timezone.now() + relativedelta(hours=5)),
'message': 'Transfer is being processed on the server',
'download_status': None,
'upload_status': None,
'status_code': None,
'function_process_id': None,
'upload_total_files': 0,
'upload_files_finished': 0,
'download_total_files': 0,
'download_files_finished': 0
}
self.process_info_path = update_or_create_process_info(
self.process_info_obj, self.action, self.ticket_number)
self.base_directory_name = '{}_{}_transfer_{}'.format(
self.source_target_name, self.destination_target_name,
self.source_resource_id)
# Remove any resources that already exist in this user's job directory
if os.path.exists(self.ticket_path):
for folder in next(os.walk(self.ticket_path))[1]:
shutil.rmtree(os.path.join(self.ticket_path, folder))
# Spawn the transfer_resource method separate from the request server by using multiprocess.
spawn_action_process(self, self._transfer_resource, self.action)
reversed_url = reverse('job_status', kwargs={'action': 'transfer'})
transfer_hyperlink = self.request.build_absolute_uri(reversed_url)
return Response(status=status.HTTP_202_ACCEPTED,
data={
'message': 'The server is processing the request.',
'transfer_job': transfer_hyperlink
})
def upload_post(self):
"""
Upload resources to a specific existing target resource or create a new target resource.
Returns
-------
Response object in JSON format
"""
# Perform token, header, target, action, and resource validation
try:
self.destination_token = get_destination_token(self.request)
self.file_duplicate_action = file_duplicate_action_validation(
self.request)
self.email = get_user_email_opt(self.request)
target_valid, self.infinite_depth = target_validation(
self.destination_target_name, self.action)
resource = file_validation(self.request)
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
self.ticket_number = hash_tokens(self.destination_token)
ticket_path = os.path.join('mediafiles', 'jobs',
str(self.ticket_number))
self.ticket_path = os.path.join('mediafiles', 'jobs',
str(self.ticket_number), 'upload')
# Remove any resources that already exist in this user's job directory
if os.path.exists(self.ticket_path):
for folder in next(os.walk(self.ticket_path))[1]:
shutil.rmtree(os.path.join(self.ticket_path, folder))
# Write process_info.json file
self.process_info_obj = {
'presqt-destination-token': hash_tokens(self.destination_token),
'status': 'in_progress',
'expiration': str(timezone.now() + relativedelta(hours=5)),
'message': 'Saving files to server and validating bag...',
'status_code': None,
'function_process_id': None,
'upload_total_files': 0,
'upload_files_finished': 0
}
self.process_info_path = update_or_create_process_info(
self.process_info_obj, self.action, self.ticket_number)
# Save files to disk and check their fixity integrity. If BagIt validation fails, attempt
# to save files to disk again. If BagIt validation fails after 3 attempts return an error.
for index in range(3):
# Extract each file in the zip file to disk
with zipfile.ZipFile(resource) as myzip:
myzip.extractall(self.ticket_path)
try:
self.base_directory_name = next(os.walk(
self.ticket_path))[1][0]
except IndexError:
self.process_info_obj['status'] = 'failure'
self.process_info_obj['expiration'] = str(timezone.now())
self.process_info_obj['status_code'] = 400
self.process_info_obj[
'message'] = 'PresQT Error: Bag is not formatted properly.'
self.process_info_path = update_or_create_process_info(
self.process_info_obj, self.action, self.ticket_number)
return Response(data={
'error':
'PresQT Error: Bag is not formatted properly.'
},
status=status.HTTP_400_BAD_REQUEST)
self.resource_main_dir = os.path.join(self.ticket_path,
self.base_directory_name)
# Validate the 'bag' and check for checksum mismatches
try:
self.bag = bagit.Bag(self.resource_main_dir)
validate_bag(self.bag)
except PresQTValidationError as e:
shutil.rmtree(self.ticket_path)
# If we've reached the maximum number of attempts then return an error response
if index == 2:
return Response(
data={'error': 'PresQT Error: {}'.format(e.data)},
status=e.status_code)
except bagit.BagError as e:
shutil.rmtree(self.ticket_path)
# If we've reached the maximum number of attempts then return an error response
if index == 2:
return Response(
data={'error': 'PresQT Error: {}'.format(e.args[0])},
status=status.HTTP_400_BAD_REQUEST)
else:
# Collect and remove any existing source metadata
try:
get_upload_source_metadata(self, self.bag)
except PresQTValidationError as e:
return Response(data={'error': e.data},
status=e.status_code)
# If the bag validated successfully then break from the loop
break
# Create a hash dictionary to compare with the hashes returned from the target after upload
# If the destination target supports a hash provided by the bag then use those hashes
# otherwise create new hashes with a target supported hash.
self.file_hashes, self.hash_algorithm = get_or_create_hashes_from_bag(
self)
# Spawn the upload_resource method separate from the request server by using multiprocess.
spawn_action_process(self, self._upload_resource, 'resource_upload')
reversed_url = reverse('job_status', kwargs={'action': 'upload'})
upload_hyperlink = self.request.build_absolute_uri(reversed_url)
return Response(status=status.HTTP_202_ACCEPTED,
data={
'message': 'The server is processing the request.',
'upload_job': upload_hyperlink
})
def transfer_get(self):
"""
Get the status of a transfer job.
"""
# Perform token validation. Read data from the process_info file.
try:
destination_token = get_destination_token(self.request)
source_token = get_source_token(self.request)
self.ticket_number = '{}_{}'.format(hash_tokens(source_token),
hash_tokens(destination_token))
self.process_data = get_process_info_data(self.ticket_number)
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
try:
transfer_process_data = self.process_data['resource_transfer_in']
except KeyError:
return Response(data={
'error':
'PresQT Error: "resource_download" not found in process_info file.'
},
status=status.HTTP_400_BAD_REQUEST)
transfer_status = transfer_process_data['status']
upload_job_percentage = calculate_job_percentage(
transfer_process_data['upload_total_files'],
transfer_process_data['upload_files_finished'])
download_job_percentage = calculate_job_percentage(
transfer_process_data['download_total_files'],
transfer_process_data['download_files_finished'])
data = {
'status_code':
transfer_process_data['status_code'],
'status':
transfer_status,
'message':
transfer_process_data['message'],
'job_percentage':
round((upload_job_percentage + download_job_percentage) / 2),
}
if transfer_status == 'finished':
http_status = status.HTTP_200_OK
data['failed_fixity'] = transfer_process_data['failed_fixity']
data['resources_ignored'] = transfer_process_data[
'resources_ignored']
data['resources_updated'] = transfer_process_data[
'resources_updated']
data['enhanced_keywords'] = transfer_process_data[
'enhanced_keywords']
data['initial_keywords'] = transfer_process_data[
'initial_keywords']
data['source_resource_id'] = transfer_process_data[
'source_resource_id']
data['destination_resource_id'] = transfer_process_data[
'destination_resource_id']
data['fairshare_evaluation_results'] = transfer_process_data[
'fairshare_evaluation_results']
data['link_to_resource'] = transfer_process_data[
'link_to_resource']
data['job_percentage'] = 99
else:
if transfer_status == 'in_progress':
http_status = status.HTTP_202_ACCEPTED
else:
http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
return Response(status=http_status, data=data)
def transfer_post(self):
"""
Transfer resources to a specific existing target resource or create a new target resource.
Returns
-------
Response object in JSON format
"""
try:
self.destination_token = get_destination_token(self.request)
self.source_token = get_source_token(self.request)
self.file_duplicate_action = file_duplicate_action_validation(
self.request)
self.source_target_name, self.source_resource_id = transfer_post_body_validation(
self.request)
target_valid, self.infinite_depth = target_validation(
self.destination_target_name, self.action)
target_validation(self.source_target_name, 'resource_transfer_out')
transfer_target_validation(self.source_target_name,
self.destination_target_name)
except PresQTValidationError as e:
return Response(data={'error': e.data}, status=e.status_code)
# Generate ticket number
ticket_number = uuid4()
self.ticket_path = os.path.join("mediafiles", "transfers",
str(ticket_number))
# Create directory and process_info json file
self.process_info_obj = {
'presqt-source-token': hash_tokens(self.source_token),
'presqt-destination-token': hash_tokens(self.destination_token),
'status': 'in_progress',
'expiration': str(timezone.now() + relativedelta(days=5)),
'message': 'Transfer is being processed on the server',
'download_status': None,
'upload_status': None,
'status_code': None,
'function_process_id': None
}
self.process_info_path = os.path.join(self.ticket_path,
"process_info.json")
write_file(self.process_info_path, self.process_info_obj, True)
self.base_directory_name = '{}_{}_transfer_{}'.format(
self.source_target_name, self.destination_target_name,
self.source_resource_id)
# Spawn the transfer_resource method separate from the request server by using multiprocess.
spawn_action_process(self, self._transfer_resource)
reversed_url = reverse('transfer_job',
kwargs={'ticket_number': ticket_number})
transfer_hyperlink = self.request.build_absolute_uri(reversed_url)
return Response(status=status.HTTP_202_ACCEPTED,
data={
'ticket_number': ticket_number,
'message': 'The server is processing the request.',
'transfer_job': transfer_hyperlink
})