def _create_file_transfer_token(
subtask_id: str,
result_package_path: str,
result_size: int,
result_package_hash: str,
authorized_client_public_key: bytes,
operation: FileTransferToken.Operation,
source_package_path: Optional[str] = None,
source_size: Optional[int] = None,
source_package_hash: Optional[str] = None,
token_expiration_deadline: Optional[int] = None,
) -> FileTransferToken:
assert (source_size and source_package_hash
and source_package_path) or (result_size and result_package_hash
and result_package_path)
assert isinstance(authorized_client_public_key, bytes)
assert isinstance(token_expiration_deadline, int) and not isinstance(
token_expiration_deadline, bool) or token_expiration_deadline is None
assert operation in [
FileTransferToken.Operation.download,
FileTransferToken.Operation.upload
]
file_transfer_token = FileTransferToken(
token_expiration_deadline=token_expiration_deadline,
storage_cluster_address=settings.STORAGE_CLUSTER_ADDRESS,
authorized_client_public_key=authorized_client_public_key,
operation=operation,
subtask_id=subtask_id)
files = []
if result_package_path and result_package_hash and result_size:
files.append(
create_file_info(
file_path=result_package_path,
package_hash=result_package_hash,
size=result_size,
category=FileTransferToken.FileInfo.Category.results,
))
if source_package_path and source_package_hash and source_size:
files.append(
create_file_info(
file_path=source_package_path,
package_hash=source_package_hash,
size=source_size,
category=FileTransferToken.FileInfo.Category.resources,
))
file_transfer_token.files = files
file_transfer_token = sign_message(file_transfer_token,
settings.CONCENT_PRIVATE_KEY)
validate_file_transfer_token(file_transfer_token)
return file_transfer_token
def test_upload_should_return_401_if_checksum_is_wrong(self):
invalid_values_with_expected_error_code = {
b'sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89':
ErrorCode.MESSAGE_FILES_CHECKSUM_WRONG_TYPE,
'': ErrorCode.MESSAGE_FILES_CHECKSUM_EMPTY,
'95a0f391c7ad86686ab1366bcd519ba5ab3cce89':
ErrorCode.MESSAGE_FILES_CHECKSUM_WRONG_FORMAT,
':95a0f391c7ad86686ab1366bcd519ba5ab3cce89':
ErrorCode.MESSAGE_FILES_CHECKSUM_INVALID_ALGORITHM,
'sha1:95a0f391c7ad86686ab1366bcd519ba5amONj':
ErrorCode.MESSAGE_FILES_CHECKSUM_INVALID_SHA1_HASH,
'sha1:': ErrorCode.MESSAGE_FILES_CHECKSUM_INVALID_SHA1_HASH,
}
for invalid_value, error_code in invalid_values_with_expected_error_code.items(
):
file1 = FileTransferToken.FileInfo(
path=
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=1024,
)
file2 = FileTransferToken.FileInfo(
path=
'blenderrr/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum=invalid_value,
size=1024,
)
self.upload_token.files = [file1, file2]
self.upload_token.sig = None
golem_upload_token = dump(self.upload_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY)
encoded_token = b64encode(golem_upload_token).decode()
response = self.client.post(
'{}{}'.format(
reverse('gatekeeper:upload'),
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend'
),
HTTP_AUTHORIZATION='Golem ' + encoded_token,
HTTP_CONCENT_AUTH=self.header_concent_auth,
content_type='application/x-www-form-urlencoded',
HTTP_X_Golem_Messages=settings.GOLEM_MESSAGES_VERSION,
)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 401)
self.assertIn('message', response.json().keys())
self.assertIn('error_code', response.json().keys())
self.assertEqual("application/json", response["Content-Type"])
self.assertEqual(response.json()["error_code"], error_code.value)
def test_upload_should_return_401_if_specific_file_info_data(self):
file = FileTransferToken.FileInfo(
path=get_storage_result_file_path(task_id=self._get_uuid(),
subtask_id=self._get_uuid()),
size=1,
checksum='sha1:356a192b7913b04c54574d18c28d46e6395428ab\n',
)
self.upload_token.files = [file]
golem_upload_token = dump(self.upload_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY)
encoded_token = b64encode(golem_upload_token).decode()
response = self.client.post(
'{}{}'.format(
reverse('gatekeeper:upload'),
get_storage_result_file_path(
task_id=self._get_uuid(),
subtask_id=self._get_uuid(),
),
),
HTTP_AUTHORIZATION='Golem ' + encoded_token,
HTTP_CONCENT_AUTH=self.header_concent_auth,
content_type='application/x-www-form-urlencoded',
HTTP_X_Golem_Messages=settings.GOLEM_MESSAGES_VERSION,
)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 401)
self.assertIn('message', response.json().keys())
self.assertEqual("application/json", response["Content-Type"])
def test_download_should_return_401_if_size_of_file_is_wrong(self):
invalid_values_with_expected_error_code = {
None: ErrorCode.MESSAGE_FILES_SIZE_EMPTY,
'number': ErrorCode.MESSAGE_FILES_SIZE_WRONG_TYPE,
'-2': ErrorCode.MESSAGE_FILES_SIZE_NEGATIVE,
'1.0': ErrorCode.MESSAGE_FILES_SIZE_WRONG_TYPE,
}
for invalid_value, error_code in invalid_values_with_expected_error_code.items(
):
file1 = FileTransferToken.FileInfo(
path=
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=1024,
)
file2 = FileTransferToken.FileInfo(
path=
'blenderrr/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=invalid_value,
)
self.download_token.files = [file1, file2]
self.download_token.sig = None
golem_download_token = dump(self.download_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY)
encoded_token = b64encode(golem_download_token).decode()
response = self.client.get(
'{}{}'.format(
reverse('gatekeeper:download'),
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend'
),
HTTP_AUTHORIZATION='Golem ' + encoded_token,
HTTP_CONCENT_AUTH=self.header_concent_auth,
)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 401)
self.assertIn('message', response.json().keys())
self.assertEqual("application/json", response["Content-Type"])
self.assertEqual(response.json()["error_code"], error_code.value)
def create_file_info(
file_path: str,
package_hash: str,
size: int,
category: FileTransferToken.FileInfo.Category,
) -> FileTransferToken.FileInfo:
assert isinstance(category, FileTransferToken.FileInfo.Category)
return FileTransferToken.FileInfo(path=file_path,
checksum=package_hash,
size=size,
category=category)
def test_upload_should_return_401_if_file_paths_are_not_unique(self):
file1 = FileTransferToken.FileInfo(
path='blender/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=1024,
)
file2 = FileTransferToken.FileInfo(
path='blender/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=1024,
)
self.upload_token.files = [file1, file2]
assert file1 == file2
golem_upload_token = dump(self.upload_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY)
encoded_token = b64encode(golem_upload_token).decode()
response = self.client.post(
'{}{}'.format(
reverse('gatekeeper:upload'),
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend'
),
HTTP_AUTHORIZATION='Golem ' + encoded_token,
HTTP_CONCENT_AUTH=self.header_concent_auth,
content_type='application/x-www-form-urlencoded',
HTTP_X_Golem_Messages=settings.GOLEM_MESSAGES_VERSION,
)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 401)
self.assertIn('message', response.json().keys())
self.assertIn('error_code', response.json().keys())
self.assertEqual("application/json", response["Content-Type"])
self.assertEqual(response.json()["error_code"],
ErrorCode.MESSAGE_FILES_PATHS_NOT_UNIQUE.value)
def test_download_should_return_401_if_file_categories_are_not_unique_across_file_info_list(
self):
file_1 = FileTransferToken.FileInfo(
path='blender/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=1024,
category=FileTransferToken.FileInfo.Category.results)
file_2 = FileTransferToken.FileInfo(
path='blender/benchmark/test_task/scene-Helicopter-28-cycles.blend',
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=1024,
category=FileTransferToken.FileInfo.Category.results)
self.download_token.files = [file_1, file_2]
golem_download_token = dump(self.download_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY)
encoded_token = b64encode(golem_download_token).decode()
response = self.client.get(
'{}{}'.format(
reverse('gatekeeper:download'),
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend'
),
HTTP_AUTHORIZATION='Golem ' + encoded_token,
HTTP_CONCENT_AUTH=self.header_concent_auth,
content_type='application/x-www-form-urlencoded',
)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 401)
self.assertIn('message', response.json().keys())
self.assertIn('error_code', response.json().keys())
self.assertEqual("application/json", response["Content-Type"])
self.assertEqual(response.json()["error_code"],
ErrorCode.MESSAGE_FILES_CATEGORY_NOT_UNIQUE.value)
def _get_auth_headers(self, file_transfer_token: FileTransferToken):
auth_key = base64.b64encode(file_transfer_token.serialize()).decode()
auth_data = base64.b64encode(
golem_messages.dump(
ClientAuthorization(
client_public_key=self.keys_auth.public_key),
self.keys_auth._private_key,
self.variant['pubkey'] # noqa pylint:disable=protected-access
)).decode()
logger.debug(
"Generating headers - ftt: %s, auth: %s, concent_auth: %s",
file_transfer_token, auth_key, auth_data)
return {
'Authorization': 'Golem ' + auth_key,
'Concent-Auth': auth_data,
}
def test_upload_should_return_401_if_newlines_in_message_path(self):
invalid_values_with_expected_error_code = {
'blen\nder/benchmark/test_task/scene-Helicopter-2\n7-cycles.blend':
ErrorCode.MESSAGE_FILES_PATH_NOT_LISTED_IN_FILES,
'\nblender/benchmark/test_task/scene-Helicopter-27-cycles.blend':
ErrorCode.MESSAGE_FILES_PATH_NOT_LISTED_IN_FILES,
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend\n':
ErrorCode.MESSAGE_FILES_PATH_NOT_LISTED_IN_FILES,
}
for value, error_code in invalid_values_with_expected_error_code.items(
):
file = FileTransferToken.FileInfo(
path=value,
checksum='sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89',
size=1024,
)
self.upload_token.files = [file]
self.upload_token.sig = None
golem_upload_token = dump(self.upload_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY)
encoded_token = b64encode(golem_upload_token).decode()
response = self.client.post(
'{}{}'.format(
reverse('gatekeeper:upload'),
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend'
),
HTTP_AUTHORIZATION='Golem ' + encoded_token,
HTTP_CONCENT_AUTH=self.header_concent_auth,
content_type='application/x-www-form-urlencoded',
HTTP_X_Golem_Messages=settings.GOLEM_MESSAGES_VERSION,
)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 401)
self.assertIn('message', response.json().keys())
self.assertIn('error_code', response.json().keys())
self.assertEqual("application/json", response["Content-Type"])
self.assertEqual(response.json()["error_code"], error_code.value)
def test_download_should_return_400_if_newlines_in_message_checksum(self):
invalid_values_with_expected_error_code = {
's\nha1:95a0f391c7ad866\n86ab1366bcd519ba5ab3cce89':
ErrorCode.MESSAGE_FILES_CHECKSUM_INVALID_ALGORITHM,
'\nsha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89':
ErrorCode.MESSAGE_FILES_CHECKSUM_INVALID_ALGORITHM,
'sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89\n':
ErrorCode.MESSAGE_FILES_CHECKSUM_INVALID_SHA1_HASH,
}
for value, error_code in invalid_values_with_expected_error_code.items(
):
file = FileTransferToken.FileInfo(
path=
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend',
checksum=value,
size=1024,
)
self.download_token.files = [file]
self.download_token.sig = None
golem_download_token = dump(self.download_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY)
encoded_token = b64encode(golem_download_token).decode()
response = self.client.get(
'{}{}'.format(
reverse('gatekeeper:download'),
'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend'
),
HTTP_AUTHORIZATION='Golem ' + encoded_token,
HTTP_CONCENT_AUTH=self.header_concent_auth,
)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 401)
self.assertIn('message', response.json().keys())
self.assertEqual("application/json", response["Content-Type"])
self.assertEqual(response.json()["error_code"], error_code.value)
def _get_download_uri(file_transfer_token: FileTransferToken,
file_category: FileTransferToken.FileInfo.Category):
return '{}{}{}'.format(
file_transfer_token.storage_cluster_address, 'download/',
file_transfer_token.get_file_info(file_category).get('path'))
def _mock_get_auth_headers(self, file_transfer_token: FileTransferToken):
return {
'Authorization': 'Golem ' + base64.b64encode(
file_transfer_token.serialize()).decode(),
}
