def test_uploading_to_exisitng_container_ignore(self):
"""
If there is an ignored file from Zenodo when uploading to an existing container, we want
to make the user aware.
"""
# 202 when uploading a new top level repo
shared_upload_function_osf(self)
# Verify the new repo exists on the PresQT Resource Collection endpoint.
url = reverse('resource_collection', kwargs={'target_name': 'zenodo'})
response_json = self.client.get(
url, **{
'HTTP_PRESQT_SOURCE_TOKEN': ZENODO_TEST_USER_TOKEN
}).json()['resources']
repo_name_list = [repo['title'] for repo in response_json]
self.assertIn(self.project_title, repo_name_list)
# Delete upload folder
shutil.rmtree(self.ticket_path)
self.resource_id = [
resource['id'] for resource in response_json
if resource['title'] == self.project_title
][0]
self.duplicate_action = 'ignore'
self.url = reverse('resource',
kwargs={
'target_name': 'zenodo',
'resource_id': self.resource_id
})
self.file = 'presqt/api_v1/tests/resources/upload/ProjectSingleFileToUpload.zip'
self.headers['HTTP_PRESQT_FILE_DUPLICATE_ACTION'] = 'ignore'
response = self.client.post(self.url,
{'presqt-file': open(self.file, 'rb')},
**self.headers)
ticket_path = 'mediafiles/jobs/{}'.format(self.ticket_number)
# Wait until the spawned off process finishes in the background
# to do validation on the resulting files
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
while process_info['resource_upload']['status'] == 'in_progress':
try:
process_info = read_file(
'{}/process_info.json'.format(ticket_path), True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
upload_job_response = self.client.get(response.data['upload_job'],
**self.headers)
self.assertEqual(upload_job_response.data['status_code'], '200')
self.assertEqual(upload_job_response.data['resources_ignored'],
['/NewProject/NewProject.presqt.zip'])
# Delete the upload folder
shutil.rmtree(ticket_path)
def test_error_bad_project_id(self):
self.url = reverse('resource', kwargs={
'target_name': 'gitlab', 'resource_id': 'badbadbadidnowaythisisreal'})
response = self.client.post(
self.url, {'presqt-file': open(self.file, 'rb')}, **self.headers)
self.ticket_path = 'mediafiles/jobs/{}'.format(self.ticket_number)
# Verify status code and message
self.assertEqual(response.status_code, 202)
self.assertEqual(
response.data['message'], 'The server is processing the request.')
# Verify process_info file status is 'in_progress' initially
process_info = read_file('{}/process_info.json'.format(self.ticket_path), True)
self.assertEqual(process_info['resource_upload']['status'], 'in_progress')
# Wait until the spawned off process finishes in the background to do further validation
process_wait(process_info, self.ticket_path)
process_info = read_file('{}/process_info.json'.format(self.ticket_path), True)
self.assertEquals(
process_info['resource_upload']['message'], "Project with id, badbadbadidnowaythisisreal, could not be found.")
self.assertEquals(process_info['resource_upload']['status_code'], 404)
shutil.rmtree(self.ticket_path)
def call_upload_resources(self):
"""
Make a POST request to `resource` to begin uploading a resource
"""
response = self.client.post(self.url,
{'presqt-file': open(self.file, 'rb')},
**self.headers)
# Verify the status code
self.assertEqual(response.status_code, 202)
self.ticket_number = response.data['ticket_number']
self.upload_job = response.data['upload_job']
self.process_info_path = 'mediafiles/uploads/{}/process_info.json'.format(
self.ticket_number)
process_info = read_file(self.process_info_path, True)
# Verify the upload_job link is what we expect
self.assertEqual(self.upload_job, ('http://testserver{}'.format(
reverse('upload_job', kwargs={'ticket_number': self.ticket_number
}))))
# Save initial process data that we can use to rewrite to the process_info file for testing
self.initial_process_info = process_info
# Wait until the spawned off process finishes in the background
while process_info['status'] == 'in_progress':
try:
process_info = read_file(self.process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
self.assertNotEqual(process_info['status'], 'in_progress')
def shared_upload_function_gitlab(test_case_instance):
test_case_instance.headers['HTTP_PRESQT_FILE_DUPLICATE_ACTION'] = test_case_instance.duplicate_action
response = test_case_instance.client.post(test_case_instance.url, {'presqt-file': open(
test_case_instance.file, 'rb')}, **test_case_instance.headers)
test_case_instance.ticket_path = 'mediafiles/jobs/{}'.format(test_case_instance.ticket_number)
# Verify status code and message
test_case_instance.assertEqual(response.status_code, 202)
test_case_instance.assertEqual(
response.data['message'], 'The server is processing the request.')
# Verify process_info file status is 'in_progress' initially
process_info = read_file('{}/process_info.json'.format(test_case_instance.ticket_path), True)
test_case_instance.assertEqual(process_info['resource_upload']['status'], 'in_progress')
# Wait until the spawned off process finishes in the background to do further validation
process_wait(process_info, test_case_instance.ticket_path)
# Verify process_info.json file data
process_info = read_file('{}/process_info.json'.format(test_case_instance.ticket_path), True)
test_case_instance.assertEqual(process_info['resource_upload']['status'], 'finished')
test_case_instance.assertEqual(process_info['resource_upload']['message'], test_case_instance.success_message)
test_case_instance.assertEqual(process_info['resource_upload']['status_code'], '200')
test_case_instance.assertEqual(process_info['resource_upload']['failed_fixity'], [])
test_case_instance.assertEqual(
process_info['resource_upload']['resources_ignored'], test_case_instance.resources_ignored)
test_case_instance.assertEqual(
process_info['resource_upload']['resources_updated'], test_case_instance.resources_updated)
test_case_instance.assertEqual(
process_info['resource_upload']['hash_algorithm'], test_case_instance.hash_algorithm)
def test_get_error_400(self):
"""
Return a 400 if the `presqt-destination-token` is missing in the headers.
"""
self.client.post(self.url, {
"source_target_name": "github",
"source_resource_id": self.resource_id,
"keywords": []}, **self.headers, format='json')
process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(self.ticket_number)
process_info = read_file(process_info_path, True)
while process_info['resource_transfer_in']['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
url = reverse('job_status', kwargs={'action': 'transfer'})
headers = {'HTTP_PRESQT_FILE_DUPLICATE_ACTION': 'ignore'}
response = self.client.get(url, **headers)
# Verify the status code and content
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data['error'],
"PresQT Error: 'presqt-destination-token' missing in the request headers.")
# Delete corresponding folder
shutil.rmtree('mediafiles/jobs/{}'.format(self.ticket_number))
def test_error_500_401(self):
"""
Return a 500 if an invalid token is provided.
"""
url = reverse('resource', kwargs={'target_name': self.target_name,
'resource_id': 'dj52w379504',
'resource_format': 'zip'})
response = self.client.get(url, **{'HTTP_PRESQT_SOURCE_TOKEN': 'eggs'})
ticket_number = response.data['ticket_number']
download_url = response.data['download_job']
process_info_path = 'mediafiles/downloads/{}/process_info.json'.format(ticket_number)
process_info = read_file(process_info_path, True)
# Adding a brief sleep to allow the download_job endpoint to not return a 202 as it loads
while process_info['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
download_response = self.client.get(download_url, **{'HTTP_PRESQT_SOURCE_TOKEN': 'eggs'})
# The endpoint lumps all errors into a 500 status code
self.assertEqual(download_response.status_code, 500)
self.assertEqual(download_response.data['status_code'], 401)
self.assertEqual(download_response.data['message'],
"Token is invalid. Response returned a 401 status code.")
# Delete corresponding folder
shutil.rmtree('mediafiles/downloads/{}'.format(ticket_number))
def test_400_error_bad_request(self):
"""
If the user attempts to post to an existing repo, return a 400.
"""
# Attempt to post to an existing repo.
self.headers[
'HTTP_PRESQT_FILE_DUPLICATE_ACTION'] = self.duplicate_action
response = self.client.post(self.url + ('209372336/'),
{'presqt-file': open(self.file, 'rb')},
**self.headers)
ticket_number = response.data['ticket_number']
ticket_path = 'mediafiles/uploads/{}'.format(ticket_number)
# Wait until the spawned off process finishes in the background
# to do validation on the resulting files
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
while process_info['status'] == 'in_progress':
try:
process_info = read_file(
'{}/process_info.json'.format(ticket_path), True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
upload_job_response = self.client.get(response.data['upload_job'],
**self.headers)
# Ensure the response is what we expect
self.assertEqual(upload_job_response.data['status_code'], 400)
self.assertEqual(upload_job_response.data['message'],
"Can't upload to an existing Github repository.")
# Delete upload folder
shutil.rmtree(ticket_path)
def test_bad_article_id(self):
"""
Return a 400 if user attempts to upload to a bad article id.
"""
url = reverse('resource',
kwargs={
'target_name': 'figshare',
'resource_id': "83375:itsbad"
})
response = self.client.post(url,
{'presqt-file': open(self.file, 'rb')},
**self.headers)
ticket_path = 'mediafiles/jobs/{}'.format(self.ticket_number)
# Wait until the spawned off process finishes in the background
# to do validation on the resulting files
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
while process_info['resource_upload']['status'] == 'in_progress':
try:
process_info = read_file(
'{}/process_info.json'.format(ticket_path), True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
upload_job_response = self.client.get(response.data['upload_job'],
**self.headers)
# Ensure the response is what we expect
self.assertEqual(upload_job_response.data['status_code'], 400)
self.assertEqual(
upload_job_response.data['message'],
"Article with id, itsbad, could not be found by the requesting user."
)
def test_get_error_500_401_token_invalid(self):
"""
Return a 500 if the BaseResource._transfer_resource method running on the server gets a 401 error because the token is invalid.
"""
self.headers['HTTP_PRESQT_DESTINATION_TOKEN'] = 'bad_token'
self.client.post(self.url, {
"source_target_name": "github",
"source_resource_id": self.resource_id, "keywords": []},
**self.headers, format='json')
self.ticket_number = '{}_{}'.format(hash_tokens(
self.source_token), hash_tokens('bad_token'))
process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(self.ticket_number)
process_info = read_file(process_info_path, True)
url = reverse('job_status', kwargs={'action': 'transfer'})
while process_info['resource_transfer_in']['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, 500)
self.assertEqual(response.data['message'],
"Token is invalid. Response returned a 401 status code.")
# Delete corresponding folder
shutil.rmtree('mediafiles/jobs/{}'.format(self.ticket_number))
def test_error_500_404(self):
"""
Return a 500 if an invalid resource_id is provided.
"""
url = reverse('resource', kwargs={'target_name': self.target_name,
'resource_id': 'bad',
'resource_format': 'zip'})
response = self.client.get(url, **self.header)
ticket_number = response.data['ticket_number']
download_url = response.data['download_job_zip']
process_info_path = 'mediafiles/downloads/{}/process_info.json'.format(ticket_number)
process_info = read_file(process_info_path, True)
while process_info['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
download_response = self.client.get(download_url, **self.header)
# The endpoint lumps all errors into a 500 status code
self.assertEqual(download_response.status_code, 500)
self.assertEqual(download_response.data['status_code'], 404)
self.assertEqual(download_response.data['message'],
"The resource with id, bad, does not exist for this user.")
# Delete corresponding folder
shutil.rmtree('mediafiles/downloads/{}'.format(ticket_number))
def test_transfer_target_keyword_error(self):
"""
This test exists to test the error raising works in target functions in enhance_keywords()
"""
github_id = "209373160:__pycache__"
self.headers['HTTP_PRESQT_KEYWORD_ACTION'] = 'automatic'
# TRANSFER RESOURCE TO OSF
response = self.client.post(self.url, {
"source_target_name": "github",
"source_resource_id": github_id,
"keywords": []},
**self.headers, format='json')
self.process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(self.ticket_number)
self.transfer_job = response.data['transfer_job']
process_info = read_file(self.process_info_path, True)
response = self.client.get(self.transfer_job, **self.headers)
self.assertEqual(response.data['message'], 'Transfer is being processed on the server')
while process_info['resource_transfer_in']['status'] == 'in_progress':
try:
process_info = read_file(self.process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
self.assertNotEqual(process_info['resource_transfer_in']['status'], 'in_progress')
# DELETE TICKET FOLDER
shutil.rmtree('mediafiles/jobs/{}'.format(self.ticket_number))
def test_error_500_404_bad_file_id(self):
"""
Return a 500 if an invalid resource_id is provided.
"""
url = reverse('resource',
kwargs={
'target_name': self.target_name,
'resource_id': '82529:12541559:not_a_file',
'resource_format': 'zip'
})
response = self.client.get(url, **self.header)
ticket_number = hash_tokens(self.token)
download_url = response.data['download_job_zip']
process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(
ticket_number)
process_info = read_file(process_info_path, True)
while process_info['resource_download']['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
download_response = self.client.get(download_url, **self.header)
# The endpoint lumps all errors into a 500 status code
self.assertEqual(download_response.status_code, 500)
self.assertEqual(download_response.data['status_code'], 404)
self.assertEqual(
download_response.data['message'],
"The resource could not be found by the requesting user.")
# Delete corresponding folder
shutil.rmtree('mediafiles/jobs/{}'.format(ticket_number))
def test_bad_resource_id(self):
"""
If the resource id given is invalid, we want to raise an error.
"""
self.url = reverse('resource',
kwargs={
'target_name': 'zenodo',
'resource_id': '52162'
})
response = self.client.post(self.url,
{'presqt-file': open(self.file, 'rb')},
**self.headers)
ticket_path = 'mediafiles/jobs/{}'.format(self.ticket_number)
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
while process_info['resource_upload']['status'] == 'in_progress':
try:
process_info = read_file(
'{}/process_info.json'.format(ticket_path), True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
details = self.client.get(response.data['upload_job'],
**self.headers).data
self.assertEqual(details['status_code'], 404)
self.assertEqual(details['message'],
"Can't find the resource with id 52162, on Zenodo")
# Delete upload folder
shutil.rmtree(ticket_path)
def test_call_transfer_success_finite_depth(self):
"""
Make a POST request to `resource` to begin transfering a resource.
"""
self.url = reverse('resource_collection',
kwargs={'target_name': 'zenodo'})
self.headers = {
'HTTP_PRESQT_DESTINATION_TOKEN': ZENODO_TEST_USER_TOKEN,
'HTTP_PRESQT_SOURCE_TOKEN': self.source_token,
'HTTP_PRESQT_FILE_DUPLICATE_ACTION': 'ignore'
}
response = self.client.post(self.url,
data={
"source_target_name": "github",
"source_resource_id": self.resource_id
},
**self.headers)
self.ticket_number = response.data['ticket_number']
self.process_info_path = 'mediafiles/transfers/{}/process_info.json'.format(
self.ticket_number)
self.transfer_job = response.data['transfer_job']
process_info = read_file(self.process_info_path, True)
self.assertEqual(self.transfer_job, ('http://testserver{}'.format(
reverse('transfer_job',
kwargs={'ticket_number': self.ticket_number}))))
response = self.client.get(self.transfer_job, **self.headers)
self.assertEqual(response.data['message'],
'Transfer is being processed on the server')
# Wait until the spawned off process finishes in the background
while process_info['status'] == 'in_progress':
try:
process_info = read_file(self.process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
# Check that transfer was successful
response = self.client.get(self.transfer_job, **self.headers)
self.assertEqual(response.data['status_code'], '200')
# Fixity errors because we're dealing with GitHub
self.assertEqual(response.data['message'],
'Transfer successful but with fixity errors.')
test_user_projects = requests.get(
'https://zenodo.org/api/deposit/depositions',
params={
'access_token': ZENODO_TEST_USER_TOKEN
}).json()
for project in test_user_projects:
if project['title'] == 'ProjectTwelve':
requests.delete(
project['links']['self'],
params={'access_token': ZENODO_TEST_USER_TOKEN})
# Delete corresponding folder
shutil.rmtree('mediafiles/transfers/{}'.format(self.ticket_number))
def test_failed_upload_to_existing_project(self):
# Mock a server error for when a put request is made.
class MockResponse:
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
mock_req = MockResponse({'error': 'The server is down.'}, 500)
# 202 when uploading a new top level repo
sleep(30)
shared_upload_function_gitlab(self)
# Verify the new repo exists on the PresQT Resource Collection endpoint.
url = reverse('resource_collection', kwargs={'target_name': 'gitlab'})
project_id = self.client.get(
url, **{'HTTP_PRESQT_SOURCE_TOKEN': GITLAB_UPLOAD_TEST_USER_TOKEN}).json()['resources'][0]['id']
self.resource_id = project_id
shutil.rmtree(self.ticket_path)
# Now I'll make an explicit call to our metadata function with a mocked server error and ensure
# it is raising an exception.
with patch('requests.post') as mock_request:
mock_request.return_value = mock_req
# Upload to the newly created project
self.url = reverse('resource', kwargs={
'target_name': 'gitlab', 'resource_id': project_id})
self.headers['HTTP_PRESQT_FILE_DUPLICATE_ACTION'] = self.duplicate_action
response = self.client.post(self.url, {'presqt-file': open(
self.file, 'rb')}, **self.headers)
ticket_number = hash_tokens(self.token)
self.ticket_path = 'mediafiles/uploads/{}'.format(ticket_number)
# Verify status code and message
self.assertEqual(response.status_code, 202)
self.assertEqual(
response.data['message'], 'The server is processing the request.')
# Verify process_info file status is 'in_progress' initially
process_info = read_file('{}/process_info.json'.format(self.ticket_path), True)
self.assertEqual(process_info['status'], 'in_progress')
# Wait until the spawned off process finishes in the background to do further validation
process_wait(process_info, self.ticket_path)
# Verify process_info.json file data
process_info = read_file('{}/process_info.json'.format(self.ticket_path), True)
self.assertEqual(process_info['status'], 'failed')
self.assertEqual(process_info['message'], "Upload failed with a status code of 500")
self.assertEqual(process_info['status_code'], 400)
# Delete upload folder
shutil.rmtree(self.ticket_path)
# Delete GitLab Project
delete_gitlab_project(project_id, GITLAB_UPLOAD_TEST_USER_TOKEN)
def shared_get_success_function_202_with_error(test_case_instance):
"""
This function will be used by tests that successfully hit the GET resource endpoint but
fail during the Resource._download_resource function.
It uses class attributes that are set in the test methods.
Parameters
----------
test_case_instance : instance
instance of a test case
"""
url = reverse('resource',
kwargs={
'target_name': test_case_instance.target_name,
'resource_id': test_case_instance.resource_id,
'resource_format': 'zip'
})
response = test_case_instance.client.get(url, **test_case_instance.header)
# Verify the status code and content
test_case_instance.assertEqual(response.status_code, 202)
test_case_instance.assertEqual(response.data['message'],
'The server is processing the request.')
ticket_number = response.data['ticket_number']
ticket_path = 'mediafiles/downloads/{}'.format(ticket_number)
# Verify process_info file status is 'in_progress' initially
process_info = read_file(
'mediafiles/downloads/{}/process_info.json'.format(ticket_number),
True)
test_case_instance.assertEqual(process_info['status'], 'in_progress')
# Wait until the spawned off process finishes in the background
# to do validation on the resulting files
while process_info['status'] == 'in_progress':
try:
process_info = read_file(
'mediafiles/downloads/{}/process_info.json'.format(
ticket_number), True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
process_info = read_file('{}/process_info.json'.format(ticket_path), True)
# Verify that the zip file doesn't exist
base_name = 'osf_download_{}'.format(test_case_instance.resource_id)
zip_path = '{}/{}.zip'.format(ticket_path, base_name)
test_case_instance.assertEqual(os.path.isfile(zip_path), False)
# Verify the final status in the process_info file is 'failed'
test_case_instance.assertEqual(process_info['status'], 'failed')
test_case_instance.assertEqual(process_info['message'],
test_case_instance.status_message)
test_case_instance.assertEqual(process_info['status_code'],
test_case_instance.status_code)
# Delete corresponding folder
shutil.rmtree(ticket_path)
def test_error_500_404(self):
"""
Return a 500 if an invalid resource_id is provided.
"""
# First we will check an invalid project id
url = reverse('resource', kwargs={'target_name': self.target_name,
'resource_id': '8219237',
'resource_format': 'zip'})
response = self.client.get(url, **self.header)
ticket_number = hash_tokens(self.token)
download_url = response.data['download_job_zip']
process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(ticket_number)
process_info = read_file(process_info_path, True)
while process_info['resource_download']['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
download_response = self.client.get(download_url, **self.header)
# The endpoint lumps all errors into a 500 status code
self.assertEqual(download_response.status_code, 500)
self.assertEqual(download_response.data['status_code'], 404)
self.assertEqual(download_response.data['message'],
"The resource with id, 8219237, does not exist for this user.")
# Delete corresponding folder
shutil.rmtree('mediafiles/jobs/{}'.format(ticket_number))
# Now we will check an invalid file id
url = reverse('resource', kwargs={'target_name': self.target_name,
'resource_id': '127eqdid-WQD2EQDWS-dw234dwd8',
'resource_format': 'zip'})
response = self.client.get(url, **self.header)
download_url = response.data['download_job_zip']
process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(ticket_number)
process_info = read_file(process_info_path, True)
while process_info['resource_download']['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
download_response = self.client.get(download_url, **self.header)
# The endpoint lumps all errors into a 500 status code
self.assertEqual(download_response.status_code, 500)
self.assertEqual(download_response.data['status_code'], 404)
self.assertEqual(download_response.data['message'],
"The resource with id, 127eqdid-WQD2EQDWS-dw234dwd8, does not exist for this user.")
# Delete corresponding folder
shutil.rmtree('mediafiles/jobs/{}'.format(ticket_number))
def test_success_bad_id_upload_existing_project(self):
"""
Return a 202 when a file is uploading.
"""
# 202 when uploading a new top level repo
shared_upload_function_github(self)
# Verify the new repo exists on the PresQT Resource Collection endpoint.
self.client.get(self.url, **{
'HTTP_PRESQT_SOURCE_TOKEN': GITHUB_TEST_USER_TOKEN
}).json()
# Delete upload folder
shutil.rmtree(self.ticket_path)
self.url = reverse('resource',
kwargs={
'target_name': 'github',
'resource_id': '58435738573489573498573498573'
})
self.headers[
'HTTP_PRESQT_FILE_DUPLICATE_ACTION'] = self.duplicate_action
response = self.client.post(self.url,
{'presqt-file': open(self.file, 'rb')},
**self.headers)
self.ticket_path = 'mediafiles/jobs/{}'.format(self.ticket_number)
# Verify status code and message
self.assertEqual(response.status_code, 202)
self.assertEqual(response.data['message'],
'The server is processing the request.')
# Verify process_info file status is 'in_progress' initially
process_info = read_file(
'{}/process_info.json'.format(self.ticket_path), True)
self.assertEqual(process_info['resource_upload']['status'],
'in_progress')
# Wait until the spawned off process finishes in the background to do further validation
process_wait(process_info, self.ticket_path)
# Verify process_info.json file data
process_info = read_file(
'{}/process_info.json'.format(self.ticket_path), True)
self.assertEqual(process_info['resource_upload']['status'], 'failed')
self.assertEqual(
process_info['resource_upload']['message'],
"The resource with id, 58435738573489573498573498573, does not exist for this user."
)
self.assertEqual(process_info['resource_upload']['status_code'], 404)
# Delete upload folder
shutil.rmtree(self.ticket_path)
def test_success_406(self):
"""
Return a 406 for unsuccessful cancel because the transfer finished already.
"""
transfer_response = self.client.post(self.url,
data={
"source_target_name":
"github",
"source_resource_id":
self.resource_id
},
**self.headers)
ticket_number = transfer_response.data['ticket_number']
ticket_path = 'mediafiles/transfers/{}'.format(ticket_number)
# Verify process_info file status is 'in_progress' initially
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
self.assertEqual(process_info['status'], 'in_progress')
# Wait until the spawned off process finishes to attempt to cancel transfer
while process_info['status'] == 'in_progress':
try:
process_info = read_file(
'{}/process_info.json'.format(ticket_path), True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
transfer_patch_url = reverse('transfer_job',
kwargs={'ticket_number': ticket_number})
transfers_patch_url_response = self.client.patch(
transfer_patch_url, **self.headers)
self.assertEquals(transfers_patch_url_response.status_code, 406)
self.assertEquals(
transfers_patch_url_response.data['message'],
"Transfer successful. Fixity can't be determined because GitHub may not have provided a file checksum. See PRESQT_FTS_METADATA.json for more details."
)
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
self.assertEquals(
process_info['message'],
"Transfer successful. Fixity can't be determined because GitHub may not have provided a file checksum. See PRESQT_FTS_METADATA.json for more details."
)
self.assertEquals(process_info['status'], 'finished')
self.assertEquals(process_info['status_code'], '200')
# Delete corresponding folder
shutil.rmtree('mediafiles/transfers/{}'.format(ticket_number))
def fairshake_assessment_validator(request, rubric_id):
"""
Perform fairshake validation for required fields.
Parameters
----------
request : HTTP request object
rubric_id: str
The ID of the rubric the requesting user would like to use
Returns
-------
Returns the answers to the rubric.
"""
rubric_answers = None
if rubric_id != '96':
try:
rubric_answers = request.data['rubric_answers']
except KeyError:
raise PresQTValidationError(
"PresQT Error: 'rubric_answers' missing in POST body.",
status.HTTP_400_BAD_REQUEST
)
# Validate that rubric answers is a dict...
if type(rubric_answers) is not dict:
raise PresQTValidationError(
"PresQT Error: 'rubric_answers' must be an object with the metric id's as the keys and answer values as the values.",
status.HTTP_400_BAD_REQUEST
)
test_translator = read_file(
'presqt/specs/services/fairshake/fairshake_test_fetch.json', True)[rubric_id]
score_translator = read_file(
'presqt/specs/services/fairshake/fairshake_score_translator.json', True)
for key, value in test_translator.items():
if key not in rubric_answers.keys():
raise PresQTValidationError(
f"Missing response for metric '{key}'. Required metrics are: {list(test_translator.keys())}",
status.HTTP_400_BAD_REQUEST)
for key, value in rubric_answers.items():
if value not in score_translator.keys():
raise PresQTValidationError(
f"'{value}' is not a valid answer. Options are: {list(score_translator.keys())}",
status.HTTP_400_BAD_REQUEST)
if key not in test_translator.keys():
raise PresQTValidationError(
f"'{key}' is not a valid metric. Required metrics are: {list(test_translator.keys())}",
status.HTTP_400_BAD_REQUEST)
return rubric_answers
def test_call_transfer_success(self):
"""
Make a POST request to `resource` to begin transferring a resource.
"""
self.headers['HTTP_PRESQT_FAIRSHARE_EVALUATOR_OPT_IN'] = 'yes'
response = self.client.post(self.url, {
"source_target_name": "github",
"source_resource_id": self.resource_id,
"keywords": []},
**self.headers,
format='json')
self.process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(self.ticket_number)
self.transfer_job = response.data['transfer_job']
process_info = read_file(self.process_info_path, True)
response = self.client.get(self.transfer_job, **self.headers)
self.assertEqual(response.data['message'], 'Transfer is being processed on the server')
# Wait until the spawned off process finishes in the background
while process_info['resource_transfer_in']['status'] == 'in_progress':
try:
process_info = read_file(self.process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
self.assertNotEqual(process_info['resource_transfer_in']['status'], 'in_progress')
# Check that transfer was successful
response = self.client.get(self.transfer_job, **self.headers)
self.assertEqual(response.data['status_code'], '200')
# Fixity errors because we're dealing with GitHub
self.assertEqual(response.data['message'],
"Transfer successful. Fixity can't be determined because GitHub may not have provided a file checksum. See PRESQT_FTS_METADATA.json for more details.")
# Ensure we have results for the 12 FAIRshare tests
self.assertEqual(len(response.data['fairshare_evaluation_results']), 12)
# Check that extra metadata was uploaded correctly
headers = {'Authorization': 'Bearer {}'.format(OSF_UPLOAD_TEST_USER_TOKEN)}
for node in requests.get('http://api.osf.io/v2/users/me/nodes', headers=headers).json()['data']:
if node['attributes']['title'] == 'Project Twelve':
self.assertEqual(node['attributes']['description'], "A test project for PresQT ")
break
# Delete corresponding folder
shutil.rmtree('mediafiles/jobs/{}'.format(self.ticket_number))
# Ensure no email was sent for this request as no email was provided.
self.assertEqual(len(mail.outbox), 0)
def test_get_error_500_500_server_error_upload(self):
"""
Return a 500 if the BaseResource._transfer_resource function running on the server gets a
500 error because the OSF server is down when attempting to transfer a project.
"""
# Create a mock response class
class MockResponse:
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
mock_req = MockResponse({'error': 'The server is down.'}, 500)
# Mock the Session POST request to return a 500 server error when creating the project
with patch('presqt.targets.osf.classes.base.OSFBase.post'
) as mock_request:
mock_request.return_value = mock_req
# Attempt to create the project (but the server is down from our mock!)
response = self.client.post(self.url,
data={
"source_target_name": "github",
"source_resource_id":
self.resource_id
},
**self.headers)
ticket_number = response.data['ticket_number']
process_info_path = 'mediafiles/transfers/{}/process_info.json'.format(
ticket_number)
process_info = read_file(process_info_path, True)
while process_info['status'] == 'in_progress':
try:
process_info = read_file(process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
# Check in on the transfer job and verify we got the 500 for the server error
url = reverse('transfer_job', kwargs={'ticket_number': ticket_number})
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, 500)
self.assertEqual(
response.data, {
'message':
"Response has status code 500 while creating project ProjectTwelve",
'status_code': 400
})
# Delete corresponding folder
shutil.rmtree('mediafiles/transfers/{}'.format(ticket_number))
def test_call_transfer_success_finite_depth(self):
"""
Make a POST request to `resource` to begin transferring a resource.
"""
self.url = reverse('resource_collection', kwargs={'target_name': 'zenodo'})
self.headers = {'HTTP_PRESQT_DESTINATION_TOKEN': ZENODO_TEST_USER_TOKEN,
'HTTP_PRESQT_SOURCE_TOKEN': self.source_token,
'HTTP_PRESQT_FILE_DUPLICATE_ACTION': 'ignore',
'HTTP_PRESQT_KEYWORD_ACTION': 'automatic',
'HTTP_PRESQT_EMAIL_OPT_IN': '*****@*****.**',
'HTTP_PRESQT_FAIRSHARE_EVALUATOR_OPT_IN': 'no'}
self.ticket_number = "{}_{}".format(hash_tokens(
self.source_token), hash_tokens(ZENODO_TEST_USER_TOKEN))
response = self.client.post(self.url, {"source_target_name": "github",
"source_resource_id": self.resource_id,
"keywords": []},
**self.headers, format='json')
self.process_info_path = 'mediafiles/jobs/{}/process_info.json'.format(self.ticket_number)
self.transfer_job = response.data['transfer_job']
process_info = read_file(self.process_info_path, True)
response = self.client.get(self.transfer_job, **self.headers)
self.assertEqual(response.data['message'], 'Transfer is being processed on the server')
# Wait until the spawned off process finishes in the background
while process_info['resource_transfer_in']['status'] == 'in_progress':
try:
process_info = read_file(self.process_info_path, True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
# Check that transfer was successful
response = self.client.get(self.transfer_job, **self.headers)
self.assertEqual(response.data['status_code'], '200')
# Fixity errors because we're dealing with GitHub
self.assertEqual(response.data['message'],
'Transfer successful but with fixity errors.')
# Ensure we did not run the 12 FAIRshare tests
self.assertEqual(response.data['fairshare_evaluation_results'], [])
test_user_projects = requests.get('https://zenodo.org/api/deposit/depositions',
params={'access_token': ZENODO_TEST_USER_TOKEN}).json()
for project in test_user_projects:
if project['title'] == 'ProjectTwelve':
requests.delete(project['links']['self'], params={
'access_token': ZENODO_TEST_USER_TOKEN})
# Delete corresponding folder
shutil.rmtree('mediafiles/jobs/{}'.format(self.ticket_number))
def test_success_200(self):
"""
Return a 200 for successful cancelled transfer process.
"""
transfer_response = self.client.post(self.url,
data={
"source_target_name":
"github",
"source_resource_id":
self.resource_id
},
**self.headers)
ticket_number = transfer_response.data['ticket_number']
ticket_path = 'mediafiles/transfers/{}'.format(ticket_number)
# Verify process_info file status is 'in_progress' initially
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
self.assertEqual(process_info['status'], 'in_progress')
# Wait until the spawned off process has a function_process_id to cancel the transfer
while not process_info['function_process_id']:
try:
process_info = read_file(
'{}/process_info.json'.format(ticket_path), True)
except json.decoder.JSONDecodeError:
# Pass while the process_info file is being written to
pass
transfer_patch_url = reverse('transfer_job',
kwargs={'ticket_number': ticket_number})
transfers_patch_url_response = self.client.patch(
transfer_patch_url, **self.headers)
self.assertEquals(transfers_patch_url_response.status_code, 200)
self.assertEquals(transfers_patch_url_response.data['message'],
'Transfer was cancelled by the user')
process_info = read_file('{}/process_info.json'.format(ticket_path),
True)
self.assertEquals(process_info['message'],
'Transfer was cancelled by the user')
self.assertEquals(process_info['status'], 'failed')
self.assertEquals(process_info['status_code'], '499')
# Delete corresponding folder
shutil.rmtree('mediafiles/transfers/{}'.format(ticket_number))
def test_error_upload_to_file(self):
"""
Test that we will get an error when attempting to upload to a file.
"""
sleep(30)
shared_upload_function_gitlab(self)
# Verify the new repo exists on the PresQT Resource Collection endpoint.
url = reverse('resource_collection', kwargs={'target_name': 'gitlab'})
project_id = self.client.get(
url, **{'HTTP_PRESQT_SOURCE_TOKEN': GITLAB_UPLOAD_TEST_USER_TOKEN}).json()['resources'][0]['id']
shutil.rmtree(self.ticket_path)
# Upload to existing repo
self.resource_id = '{}:funnyfunnyimages%2FScreen Shot 2019-07-15 at 3%2E26%2E49 PM%2Epng'.format(
project_id)
self.url = reverse('resource', kwargs={
'target_name': 'gitlab', 'resource_id': self.resource_id})
self.headers['HTTP_PRESQT_FILE_DUPLICATE_ACTION'] = self.duplicate_action
response = self.client.post(self.url, {'presqt-file': open(
self.file, 'rb')}, **self.headers)
ticket_number = hash_tokens(self.token)
self.ticket_path = 'mediafiles/uploads/{}'.format(ticket_number)
# Verify status code and message
self.assertEqual(response.status_code, 202)
self.assertEqual(
response.data['message'], 'The server is processing the request.')
# Verify process_info file status is 'in_progress' initially
process_info = read_file('{}/process_info.json'.format(self.ticket_path), True)
self.assertEqual(process_info['status'], 'in_progress')
# Wait until the spawned off process finishes in the background to do further validation
process_wait(process_info, self.ticket_path)
# Verify process_info.json file data
process_info = read_file('{}/process_info.json'.format(self.ticket_path), True)
self.assertEqual(process_info['status'], 'failed')
self.assertEqual(
process_info['message'], 'Resource with id, {}, belongs to a file.'.format(self.resource_id))
self.assertEqual(process_info['status_code'], 400)
shutil.rmtree(self.ticket_path)
# Delete upload folder and project
delete_gitlab_project(project_id, GITLAB_UPLOAD_TEST_USER_TOKEN)
def target_validation(target_name, action):
"""
Given a Target name and an action, determine if the target_name is a valid
target in target.json and if the target supports the action.
Parameters
----------
target_name : str
Name of the Target.
action : str
Type of action the API is looking to perform on the Target
Returns
-------
True if the validation passes.
Raises a custom ValidationException error if validation fails.
"""
json_data = read_file('presqt/specs/targets.json', True)
for data in json_data:
if data['name'] == target_name:
if data["supported_actions"][action] is False:
raise PresQTValidationError(
"PresQT Error: '{}' does not support the action '{}'.".
format(target_name, action), status.HTTP_400_BAD_REQUEST)
return True, data['infinite_depth']
else:
raise PresQTValidationError(
"PresQT Error: '{}' is not a valid Target name.".format(
target_name), status.HTTP_404_NOT_FOUND)
def get(self, request):
"""
Returns the list of tests available to the user.
Returns
-------
200: OK
[
{
"test_name": "FAIR Metrics Gen2- Unique Identifier "
"description": "Metric to test if the metadata resource has a unique identifier. This is done by comparing the GUID to the patterns (by regexp) of known GUID schemas such as URLs and DOIs. Known schema are registered in FAIRSharing (https://fairsharing.org/standards/?q=&selected_facets=type_exact:identifier%20schema)",
"test_id": 1
},
{
"test_name": "FAIR Metrics Gen2 - Identifier Persistence "
"description": "Metric to test if the unique identifier of the metadata resource is likely to be persistent. Known schema are registered in FAIRSharing (https://fairsharing.org/standards/?q=&selected_facets=type_exact:identifier%20schema). For URLs that don't follow a schema in FAIRSharing we test known URL persistence schemas (purl, oclc, fdlp, purlz, w3id, ark).",
"test_id": 2
}...
]
"""
fairshare_test_info = read_file(
"presqt/specs/services/fairshare/fairshare_description_fetch.json",
True)
test_list = [{
"test_name": value['test_name'],
"description": value['description'],
"test_id": int(key.rpartition("/")[2])
} for key, value in fairshare_test_info.items()]
return Response(status=status.HTTP_200_OK, data=test_list)
def test_success_202(self):
"""
Return a 202 if the resource has not finished being prepared on the server.
"""
shared_call_get_resource_zip(self, self.resource_id)
# Update the fixity_info.json to say the resource hasn't finished processing
write_file(self.process_info_path, self.initial_process_info, True)
url = reverse('download_job',
kwargs={'ticket_number': self.ticket_number})
response = self.client.get(url, **self.header)
# Verify the status code and content
self.assertEqual(response.status_code, 202)
self.assertEqual(
response.data, {
'message': 'Download is being processed on the server',
'status_code': None
})
# Verify the status of the process_info file is 'in_progress'
process_info = read_file(self.process_info_path, True)
self.assertEqual(process_info['status'], 'in_progress')
# Delete corresponding folder
shutil.rmtree('mediafiles/downloads/{}'.format(self.ticket_number))
def update_or_create_process_info(process_obj, action, ticket_number):
"""
Create or update the process_info.json file for a job.
Parameters
----------
process_obj: dict
Process info dictionary to save in the process_info.json file
action: str
The current action which the process_obj will be saved to in the process_info.json file
ticket_number: str
Ticket number for user's action and also the name of the directory for process_info.json
Returns
-------
Returns the path to the process_info.json file
"""
process_info_path = os.path.join('mediafiles', 'jobs', str(ticket_number),
'process_info.json')
# If there already exists a process_info.json file for this user then add to the process dict
if os.path.isfile(process_info_path):
file_obj = read_file(process_info_path, True)
file_obj[action] = process_obj
# If no process_info.json file exists for this user than create a new process dict
else:
file_obj = {action: process_obj}
write_file(process_info_path, file_obj, True)
return process_info_path
def handle(self, *args, **kwargs):
"""
Delete all mediafiles that have run past their expiration date.
"""
if os.environ['ENVIRONMENT'] == 'development':
print('***delete_outdated_mediafiles is running in development mode.***')
directories_list = [
'/usr/src/app/mediafiles/jobs/*/',
'/usr/src/app/mediafiles/bag_tool/*/'
]
directories = []
[directories.extend(glob(directory)) for directory in directories_list]
for directory in directories:
try:
data = read_file('{}process_info.json'.format(directory), True)
except (FileNotFoundError, KeyError):
shutil.rmtree(directory)
print('{} has been deleted. No process_info.json file found'.format(directory))
else:
for key, value in data.items():
if 'expiration' in value.keys():
if parse(value['expiration']) <= timezone.now() or os.environ['ENVIRONMENT'] == 'development':
shutil.rmtree(directory)
print('{} has been deleted.'.format(directory))
break
else:
if os.environ['ENVIRONMENT'] == 'development':
shutil.rmtree(directory)
print('{} has been deleted.'.format(directory))
else:
print('{} has been retained.'.format(directory))
