async def read(self, organization_id: OrganizationID, id: UUID) -> bytes:
slug = f"{organization_id}/{id}"
try:
obj = self._s3.get_object(Bucket=self._s3_bucket, Key=slug)
except S3ClientError as exc:
if exc.response["Error"]["Code"] == "404":
raise BlockNotFoundError() from exc
else:
raise BlockTimeoutError() from exc
except S3EndpointConnectionError as exc:
raise BlockTimeoutError() from exc
return obj["Body"].read()
async def read(self, organization_id: OrganizationID, id: UUID) -> bytes:
slug = f"{organization_id}/{id}"
try:
headers, obj = await trio.to_thread.run_sync(
self.swift_client.get_object, self._container, slug)
except ClientException as exc:
if exc.http_status == 404:
raise BlockNotFoundError() from exc
else:
raise BlockTimeoutError() from exc
return obj
async def create(self, organization_id: OrganizationID, id: UUID,
block: bytes) -> None:
slug = f"{organization_id}/{id}"
try:
await trio.to_thread.run_sync(
partial(self._s3.head_object, Bucket=self._s3_bucket,
Key=slug))
except S3ClientError as exc:
if exc.response["Error"]["Code"] == "404":
try:
await trio.to_thread.run_sync(
partial(self._s3.put_object,
Bucket=self._s3_bucket,
Key=slug,
Body=block))
except (S3ClientError, S3EndpointConnectionError) as exc:
raise BlockTimeoutError() from exc
else:
raise BlockTimeoutError() from exc
except S3EndpointConnectionError as exc:
raise BlockTimeoutError() from exc
else:
raise BlockAlreadyExistsError()
async def create(self, organization_id: OrganizationID, id: UUID,
block: bytes) -> None:
slug = f"{organization_id}/{id}"
try:
_, obj = await trio.to_thread.run_sync(
self.swift_client.get_object, self._container, slug)
except ClientException as exc:
if exc.http_status == 404:
await trio.to_thread.run_sync(
partial(self.swift_client.put_object, self._container,
slug, block))
else:
raise BlockTimeoutError() from exc
else:
raise BlockAlreadyExistsError()
async def create(self, organization_id: OrganizationID, id: UUID,
block: bytes) -> None:
nb_chunks = len(self.blockstores) - 1
chunks = split_block_in_chunks(block, nb_chunks)
assert len(chunks) == nb_chunks
checksum_chunk = generate_checksum_chunk(chunks)
# Actually do the upload
error_count = 0
async def _subblockstore_create(nursery, blockstore_index,
chunk_or_checksum):
nonlocal error_count
try:
await self.blockstores[blockstore_index].create(
organization_id, id, chunk_or_checksum)
except BlockAlreadyExistsError:
# It's possible a previous tentative to upload this block has
# failed due to another blockstore not available. In such case
# a retrial will raise AlreadyExistsError on all the blockstores
# that sucessfully uploaded the block during last attempt.
# Only solution to solve this is to ignore AlreadyExistsError.
pass
except BlockTimeoutError as exc:
error_count += 1
logger.warning(
f"Cannot reach RAID5 blockstore #{blockstore_index} to create block {id}",
exc_info=exc,
)
if error_count > 1:
# Early exit
nursery.cancel_scope.cancel()
async with trio.open_service_nursery() as nursery:
for i, chunk_or_checksum in enumerate([*chunks, checksum_chunk]):
nursery.start_soon(_subblockstore_create, nursery, i,
chunk_or_checksum)
if error_count > 1:
# Only a single blockstore is allowed to fail
logger.error(
f"Block {id} cannot be created: Too many failing blockstores in the RAID5 cluster"
)
raise BlockTimeoutError(
"More than 1 blockstores has failed in the RAID5 cluster")
async def mock_read(organization_id, id):
await trio.sleep(0)
raise BlockTimeoutError()
async def mock_create(organization_id, id, block):
await trio.sleep(0)
raise BlockTimeoutError()
async def read(self, organization_id: OrganizationID, id: UUID) -> bytes:
timeout_count = 0
fetch_results = [None] * len(self.blockstores)
async def _partial_blockstore_read(nursery, blockstore_index):
nonlocal timeout_count
nonlocal fetch_results
try:
fetch_results[blockstore_index] = await self.blockstores[
blockstore_index].read(organization_id, id)
except BlockNotFoundError as exc:
# We don't know yet if this id doesn't exists globally or only in this blockstore...
fetch_results[blockstore_index] = exc
except BlockTimeoutError as exc:
fetch_results[blockstore_index] = exc
timeout_count += 1
logger.warning(
f"Cannot reach RAID5 blockstore #{blockstore_index} to read block {id}",
exc_info=exc,
)
if timeout_count > 1:
nursery.cancel_scope.cancel()
else:
# Try to fetch the checksum to rebuild the current missing chunk...
nursery.start_soon(_partial_blockstore_read, nursery,
len(self.blockstores) - 1)
async with trio.open_service_nursery() as nursery:
# Don't fetch the checksum by default
for blockstore_index in range(len(self.blockstores) - 1):
nursery.start_soon(_partial_blockstore_read, nursery,
blockstore_index)
if timeout_count == 0:
# Sanity check: no errors and we didn't fetch the checksum
assert len([res for res in fetch_results if res is None]) == 1
assert fetch_results[-1] is None
assert not len(
[res for res in fetch_results if isinstance(res, Exception)])
return rebuild_block_from_chunks(fetch_results[:-1], None)
elif timeout_count == 1:
checksum = fetch_results[-1]
# Sanity check: one error and we have fetched the checksum
assert len([res for res in fetch_results if res is None]) == 0
assert isinstance(checksum, (bytes, bytearray))
assert len([
res for res in fetch_results if isinstance(res, Exception)
]) == 1
return rebuild_block_from_chunks(
[
res if isinstance(res, (bytes, bytearray)) else None
for res in fetch_results[:-1]
],
checksum,
)
else:
logger.error(
f"Block {id} cannot be read: Too many failing blockstores in the RAID5 cluster"
)
raise BlockTimeoutError(
"More than 1 blockstores has failed in the RAID5 cluster")
