def _test_file_get_invalid_token(self, replica: Replica, scheme: str,
test_bucket: str, uploader: Uploader):
src_key = generate_test_key()
src_data = os.urandom(1024)
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key, "text/plain")
source_url = f"{scheme}://{test_bucket}/{src_key}"
file_uuid = str(uuid.uuid4())
bundle_uuid = str(uuid.uuid4())
version = datetime_to_version_format(datetime.datetime.utcnow())
# should be able to do this twice (i.e., same payload, different UUIDs)
self.upload_file(source_url,
file_uuid,
bundle_uuid=bundle_uuid,
version=version)
url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query(
"replica",
replica.name).add_query("version",
version).add_query("token", "{}"))
@eventually(30, 0.1)
def try_get():
self.assertGetResponse(url,
requests.codes.bad_request,
headers=get_auth_header())
try_get()
def _test_file_put(self, replica: Replica, scheme: str, test_bucket: str, uploader: Uploader):
src_key = generate_test_key()
src_data = os.urandom(1024)
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key, "text/plain")
source_url = f"{scheme}://{test_bucket}/{src_key}"
file_uuid = str(uuid.uuid4())
bundle_uuid = str(uuid.uuid4())
version = datetime_to_version_format(datetime.datetime.utcnow())
# should be able to do this twice (i.e., same payload, different UUIDs)
self.upload_file(source_url, file_uuid, bundle_uuid=bundle_uuid, version=version)
self.upload_file(source_url, str(uuid.uuid4()))
# should be able to do this twice (i.e., same payload, same UUIDs)
self.upload_file(source_url, file_uuid, bundle_uuid=bundle_uuid,
version=version, expected_code=requests.codes.ok)
# should *NOT* be able to do this twice (i.e., different payload, same UUIDs)
self.upload_file(source_url, file_uuid, version=version, expected_code=requests.codes.conflict)
def _test_file_put_cached(self, replica: Replica, scheme: str,
test_bucket: str, test_checkout_bucket: str,
uploader: Uploader):
stored_cache_criteria = os.environ.get('CHECKOUT_CACHE_CRITERIA')
try:
os.environ[
'CHECKOUT_CACHE_CRITERIA'] = '[{"type":"application/json","max_size":12314}]'
handle = Config.get_blobstore_handle(replica)
src_key = generate_test_key()
src_data = b'{"status":"valid"}'
source_url = f"{scheme}://{test_bucket}/{src_key}"
file_uuid = str(uuid.uuid4())
bundle_uuid = str(uuid.uuid4())
version = datetime_to_version_format(datetime.datetime.utcnow())
# write dummy file and upload to upload area
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key,
"application/json")
# upload file to DSS
self.upload_file(source_url,
file_uuid,
bundle_uuid=bundle_uuid,
version=version)
metadata = handle.get_user_metadata(test_bucket, src_key)
dst_key = ("blobs/" + ".".join([
metadata['hca-dss-sha256'], metadata['hca-dss-sha1'],
metadata['hca-dss-s3_etag'], metadata['hca-dss-crc32c']
])).lower()
for wait_to_upload_into_checkout_bucket in range(30):
try:
# get uploaded blob key from the checkout bucket
file_metadata = json.loads(
handle.get(test_checkout_bucket,
dst_key).decode("utf-8"))
break
except BlobNotFoundError:
time.sleep(1)
else:
file_metadata = json.loads(
handle.get(test_checkout_bucket, dst_key).decode("utf-8"))
assert file_metadata[
"status"] == "valid" # the file exists in the checkout bucket
finally:
os.environ['CHECKOUT_CACHE_CRITERIA'] = stored_cache_criteria
def _test_file_size(self, replica: Replica, scheme: str, test_bucket: str,
uploader: Uploader):
src_key = generate_test_key()
src_size = 1024 + int.from_bytes(os.urandom(1), byteorder='little')
src_data = os.urandom(src_size)
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key, "text/plain")
source_url = f"{scheme}://{test_bucket}/{src_key}"
file_uuid = str(uuid.uuid4())
bundle_uuid = str(uuid.uuid4())
version = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H%M%S.%fZ")
self.upload_file(source_url,
file_uuid,
bundle_uuid=bundle_uuid,
version=version)
url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query(
"replica", replica.name))
with override_bucket_config(BucketConfig.TEST):
resp_obj = self.assertGetResponse(
url,
requests.codes.found,
headers=get_auth_header(),
redirect_follow_retries=FILE_GET_RETRY_COUNT,
min_retry_interval_header=RETRY_AFTER_INTERVAL,
override_retry_interval=1,
)
url = resp_obj.response.headers['Location']
data = requests.get(url)
self.assertEqual(len(data.content), src_size)
self.assertEqual(resp_obj.response.headers['X-DSS-SIZE'],
str(src_size))
def _test_file_size(self, replica: Replica, scheme: str, test_bucket: str, uploader: Uploader):
src_key = generate_test_key()
src_size = 1024 + int.from_bytes(os.urandom(1), byteorder='little')
src_data = os.urandom(src_size)
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key, "text/plain")
source_url = f"{scheme}://{test_bucket}/{src_key}"
file_uuid = str(uuid.uuid4())
bundle_uuid = str(uuid.uuid4())
version = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H%M%S.%fZ")
self.upload_file(source_url, file_uuid, bundle_uuid=bundle_uuid, version=version)
url = str(UrlBuilder()
.set(path="/v1/files/" + file_uuid)
.add_query("replica", replica.name))
for i in range(FILE_GET_RETRY_COUNT):
with override_bucket_config(BucketConfig.TEST):
resp_obj = self.assertGetResponse(
url,
[requests.codes.found, requests.codes.moved]
)
if resp_obj.response.status_code == requests.codes.found:
url = resp_obj.response.headers['Location']
data = requests.get(url)
self.assertEqual(len(data.content), src_size)
self.assertEqual(resp_obj.response.headers['X-DSS-SIZE'], str(src_size))
return
elif resp_obj.response.status_code == requests.codes.moved:
retryAfter = int(resp_obj.response.headers['Retry-After'])
self.assertEqual(retryAfter, RETRY_AFTER_INTERVAL)
self.assertIn(url, resp_obj.response.headers['Location'])
self.fail(f"Failed after {FILE_GET_RETRY_COUNT} retries.")
def _test_file_get_checkout(self, replica: Replica, scheme: str,
test_bucket: str, uploader: Uploader):
handle = Config.get_blobstore_handle(replica)
src_key = generate_test_key()
src_data = os.urandom(1024)
source_url = f"{scheme}://{test_bucket}/{src_key}"
file_uuid = str(uuid.uuid4())
bundle_uuid = str(uuid.uuid4())
version = datetime_to_version_format(datetime.datetime.utcnow())
# write dummy file and upload to upload area
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key, "text/plain")
# upload file to DSS
self.upload_file(source_url,
file_uuid,
bundle_uuid=bundle_uuid,
version=version)
url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query(
"replica", replica.name).add_query("version", version))
# get uploaded blob key
file_metadata = json.loads(
handle.get(test_bucket,
f"files/{file_uuid}.{version}").decode("utf-8"))
file_key = compose_blob_key(file_metadata)
@eventually(20, 1)
def test_checkout():
# assert 302 and verify checksum on checkout completion
api_get = self.assertGetResponse(url,
requests.codes.found,
headers=get_auth_header(),
redirect_follow_retries=0)
file_get = requests.get(api_get.response.headers['Location'])
self.assertTrue(file_get.ok)
self.assertEquals(file_get.content, src_data)
with self.subTest(
f"{replica}: Initiates checkout and returns 301 for GET on 'uncheckedout' file."
):
# assert 301 redirect on first GET
self.assertGetResponse(url,
requests.codes.moved,
headers=get_auth_header(),
redirect_follow_retries=0)
test_checkout()
with self.subTest(
f"{replica}: Initiates checkout and returns 301 for GET on nearly expired checkout file."
):
now = datetime.datetime.now(datetime.timezone.utc)
creation_date_fn = (
"cloud_blobstore.s3.S3BlobStore.get_creation_date"
if replica.name == "aws" else
"cloud_blobstore.gs.GSBlobStore.get_creation_date")
with mock.patch(creation_date_fn) as mock_creation_date:
blob_ttl_days = int(os.environ['DSS_BLOB_TTL_DAYS'])
mock_creation_date.return_value = now - datetime.timedelta(
days=blob_ttl_days, hours=1, minutes=5)
self.assertGetResponse(url,
requests.codes.moved,
headers=get_auth_header(),
redirect_follow_retries=0)
test_checkout()
with self.subTest(
f"{replica}: Initiates checkout and returns 302 immediately for GET on stale checkout file."
):
now = datetime.datetime.now(datetime.timezone.utc)
creation_date = handle.get_creation_date(replica.checkout_bucket,
file_key)
creation_date_fn = (
"cloud_blobstore.s3.S3BlobStore.get_creation_date"
if replica.name == "aws" else
"cloud_blobstore.gs.GSBlobStore.get_creation_date")
with mock.patch(creation_date_fn) as mock_creation_date:
# assert 302 found on stale file and that last modified refreshes
blob_ttl_days = int(os.environ['DSS_BLOB_PUBLIC_TTL_DAYS'])
mock_creation_date.return_value = now - datetime.timedelta(
days=blob_ttl_days + 1)
self.assertGetResponse(url,
requests.codes.found,
headers=get_auth_header(),
redirect_follow_retries=0)
self.assertTrue(
creation_date > handle.get_creation_date(
replica.checkout_bucket, file_key),
f'\ncurr_creation_date: {creation_date}'
f'\nprev_creation_date: {handle.get_creation_date(replica.checkout_bucket)}'
)
handle.delete(test_bucket, f"files/{file_uuid}.{version}")
handle.delete(replica.checkout_bucket, file_key)
def _test_file_put(self, replica: Replica, scheme: str, test_bucket: str,
uploader: Uploader):
src_key = generate_test_key()
src_data = os.urandom(1024)
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key, "text/plain")
source_url = f"{scheme}://{test_bucket}/{src_key}"
file_uuid = str(uuid.uuid4())
bundle_uuid = str(uuid.uuid4())
version = datetime_to_version_format(datetime.datetime.utcnow())
self._test_put_auth_errors(scheme, test_bucket)
with self.subTest(
f"{replica}: Created returned when uploading a file with a unique payload, and FQID"
):
self.upload_file(source_url,
file_uuid,
bundle_uuid=bundle_uuid,
version=version)
with self.subTest(
f"{replica}: Created returned when uploading a file with same payload, and different FQID"
):
self.upload_file(source_url, str(uuid.uuid4()))
with self.subTest(
f"{replica}: OK returned when uploading a file with the same payload, UUID, version"
):
self.upload_file(source_url,
file_uuid,
bundle_uuid=bundle_uuid,
version=version,
expected_code=requests.codes.ok)
with self.subTest(
f"{replica}: Conflict returned when uploading a file with a different payload and same FQID"
):
src_key_temp = generate_test_key()
src_data_temp = os.urandom(128)
with tempfile.NamedTemporaryFile(delete=True) as fh:
fh.write(src_data_temp)
fh.flush()
uploader.checksum_and_upload_file(fh.name, src_key_temp,
"text/plain")
source_url_temp = f"{scheme}://{test_bucket}/{src_key_temp}"
self.upload_file(source_url_temp,
file_uuid,
version=version,
expected_code=requests.codes.conflict)
with self.subTest(
f"{replica}: Bad returned when uploading a file with an invalid version"
):
self.upload_file(source_url,
file_uuid,
version='',
expected_code=requests.codes.bad_request)
invalid_version = 'ABCD'
with self.subTest(
f"{replica}: bad_request returned "
f"when uploading a file with invalid version {invalid_version}"
):
self.upload_file(source_url,
file_uuid,
version=invalid_version,
expected_code=requests.codes.bad_request)
with self.subTest(
f"{replica}: Bad returned when uploading a file without a version"
):
self.upload_file(source_url,
file_uuid,
version='missing',
expected_code=requests.codes.bad_request)
invalid_uuids = ['ABCD', '1234']
for invalid_uuid in invalid_uuids:
with self.subTest(
f"{replica}: Bad returned "
f"when uploading a file with invalid UUID {invalid_uuid}"):
self.upload_file(source_url,
invalid_uuid,
expected_code=requests.codes.bad_request)
with self.subTest(
f"{replica}: forbidden returned "
f"when uploading a file with without UUID {invalid_uuid}"):
self.upload_file(source_url,
'',
expected_code=requests.codes.forbidden)