async def read(self, organization_id: OrganizationID, id: UUID) -> bytes: slug = f"{organization_id}/{id}" try: obj = self._s3.get_object(Bucket=self._s3_bucket, Key=slug) except S3ClientError as exc: if exc.response["Error"]["Code"] == "404": raise BlockNotFoundError() from exc else: raise BlockTimeoutError() from exc except S3EndpointConnectionError as exc: raise BlockTimeoutError() from exc return obj["Body"].read()
async def read(self, organization_id: OrganizationID, id: UUID) -> bytes: slug = f"{organization_id}/{id}" try: headers, obj = await trio.to_thread.run_sync( self.swift_client.get_object, self._container, slug) except ClientException as exc: if exc.http_status == 404: raise BlockNotFoundError() from exc else: raise BlockTimeoutError() from exc return obj
async def create(self, organization_id: OrganizationID, id: UUID, block: bytes) -> None: slug = f"{organization_id}/{id}" try: await trio.to_thread.run_sync( partial(self._s3.head_object, Bucket=self._s3_bucket, Key=slug)) except S3ClientError as exc: if exc.response["Error"]["Code"] == "404": try: await trio.to_thread.run_sync( partial(self._s3.put_object, Bucket=self._s3_bucket, Key=slug, Body=block)) except (S3ClientError, S3EndpointConnectionError) as exc: raise BlockTimeoutError() from exc else: raise BlockTimeoutError() from exc except S3EndpointConnectionError as exc: raise BlockTimeoutError() from exc else: raise BlockAlreadyExistsError()
async def create(self, organization_id: OrganizationID, id: UUID, block: bytes) -> None: slug = f"{organization_id}/{id}" try: _, obj = await trio.to_thread.run_sync( self.swift_client.get_object, self._container, slug) except ClientException as exc: if exc.http_status == 404: await trio.to_thread.run_sync( partial(self.swift_client.put_object, self._container, slug, block)) else: raise BlockTimeoutError() from exc else: raise BlockAlreadyExistsError()
async def create(self, organization_id: OrganizationID, id: UUID, block: bytes) -> None: nb_chunks = len(self.blockstores) - 1 chunks = split_block_in_chunks(block, nb_chunks) assert len(chunks) == nb_chunks checksum_chunk = generate_checksum_chunk(chunks) # Actually do the upload error_count = 0 async def _subblockstore_create(nursery, blockstore_index, chunk_or_checksum): nonlocal error_count try: await self.blockstores[blockstore_index].create( organization_id, id, chunk_or_checksum) except BlockAlreadyExistsError: # It's possible a previous tentative to upload this block has # failed due to another blockstore not available. In such case # a retrial will raise AlreadyExistsError on all the blockstores # that sucessfully uploaded the block during last attempt. # Only solution to solve this is to ignore AlreadyExistsError. pass except BlockTimeoutError as exc: error_count += 1 logger.warning( f"Cannot reach RAID5 blockstore #{blockstore_index} to create block {id}", exc_info=exc, ) if error_count > 1: # Early exit nursery.cancel_scope.cancel() async with trio.open_service_nursery() as nursery: for i, chunk_or_checksum in enumerate([*chunks, checksum_chunk]): nursery.start_soon(_subblockstore_create, nursery, i, chunk_or_checksum) if error_count > 1: # Only a single blockstore is allowed to fail logger.error( f"Block {id} cannot be created: Too many failing blockstores in the RAID5 cluster" ) raise BlockTimeoutError( "More than 1 blockstores has failed in the RAID5 cluster")
async def mock_read(organization_id, id): await trio.sleep(0) raise BlockTimeoutError()
async def mock_create(organization_id, id, block): await trio.sleep(0) raise BlockTimeoutError()
async def read(self, organization_id: OrganizationID, id: UUID) -> bytes: timeout_count = 0 fetch_results = [None] * len(self.blockstores) async def _partial_blockstore_read(nursery, blockstore_index): nonlocal timeout_count nonlocal fetch_results try: fetch_results[blockstore_index] = await self.blockstores[ blockstore_index].read(organization_id, id) except BlockNotFoundError as exc: # We don't know yet if this id doesn't exists globally or only in this blockstore... fetch_results[blockstore_index] = exc except BlockTimeoutError as exc: fetch_results[blockstore_index] = exc timeout_count += 1 logger.warning( f"Cannot reach RAID5 blockstore #{blockstore_index} to read block {id}", exc_info=exc, ) if timeout_count > 1: nursery.cancel_scope.cancel() else: # Try to fetch the checksum to rebuild the current missing chunk... nursery.start_soon(_partial_blockstore_read, nursery, len(self.blockstores) - 1) async with trio.open_service_nursery() as nursery: # Don't fetch the checksum by default for blockstore_index in range(len(self.blockstores) - 1): nursery.start_soon(_partial_blockstore_read, nursery, blockstore_index) if timeout_count == 0: # Sanity check: no errors and we didn't fetch the checksum assert len([res for res in fetch_results if res is None]) == 1 assert fetch_results[-1] is None assert not len( [res for res in fetch_results if isinstance(res, Exception)]) return rebuild_block_from_chunks(fetch_results[:-1], None) elif timeout_count == 1: checksum = fetch_results[-1] # Sanity check: one error and we have fetched the checksum assert len([res for res in fetch_results if res is None]) == 0 assert isinstance(checksum, (bytes, bytearray)) assert len([ res for res in fetch_results if isinstance(res, Exception) ]) == 1 return rebuild_block_from_chunks( [ res if isinstance(res, (bytes, bytearray)) else None for res in fetch_results[:-1] ], checksum, ) else: logger.error( f"Block {id} cannot be read: Too many failing blockstores in the RAID5 cluster" ) raise BlockTimeoutError( "More than 1 blockstores has failed in the RAID5 cluster")