async def test_s3_read():
with mock.patch("boto3.client") as client_mock:
client_mock.return_value = Mock()
client_mock().head_bucket.return_value = True
blockstore = S3BlockStoreComponent("europe", "parsec", "john",
"secret")
# Ok
response_mock = Mock()
response_mock.read.return_value = "content"
client_mock().get_object.return_value = {"Body": response_mock}
assert await blockstore.read("org42", 123) == "content"
# Not found
client_mock().get_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "404"
}}, operation_name="GET")
with pytest.raises(BlockNotFoundError):
assert await blockstore.read("org42", 123)
# Connection error
client_mock().get_object.side_effect = S3EndpointConnectionError(
endpoint_url="url")
with pytest.raises(BlockTimeoutError):
assert await blockstore.read("org42", 123)
# Unknown exception
client_mock().get_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "401"
}}, operation_name="GET")
with pytest.raises(BlockTimeoutError):
assert await blockstore.read("org42", 123)
async def test_s3_read(caplog):
org_id = OrganizationID("org42")
block_id = BlockID.from_hex("0694a21176354e8295e28a543e5887f9")
def _assert_log():
log = caplog.assert_occured_once("[warning ] Block read error")
assert f"organization_id={org_id}" in log
assert f"block_id={block_id}" in log
assert len(caplog.messages) == 1
caplog.clear()
with mock.patch("boto3.client") as client_mock:
client_mock.return_value = Mock()
client_mock().head_bucket.return_value = True
blockstore = S3BlockStoreComponent("europe", "parsec", "john",
"secret")
# Ok
response_mock = Mock()
response_mock.read.return_value = "content"
client_mock().get_object.return_value = {"Body": response_mock}
assert await blockstore.read(org_id, block_id) == "content"
client_mock().get_object.assert_called_once_with(
Bucket="parsec", Key="org42/0694a211-7635-4e82-95e2-8a543e5887f9")
client_mock().get_object.reset_mock()
assert not caplog.messages
# Not found
client_mock().get_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "404"
}}, operation_name="GET")
with pytest.raises(BlockStoreError):
assert await blockstore.read(org_id, block_id)
_assert_log()
# Connection error
client_mock().get_object.side_effect = S3EndpointConnectionError(
endpoint_url="url")
with pytest.raises(BlockStoreError):
assert await blockstore.read(org_id, block_id)
_assert_log()
# Unknown exception
client_mock().get_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "401"
}}, operation_name="GET")
with pytest.raises(BlockStoreError):
assert await blockstore.read(org_id, block_id)
_assert_log()
async def test_s3_create():
with mock.patch("boto3.client") as client_mock:
client_mock.return_value = Mock()
client_mock().head_container.return_value = True
blockstore = S3BlockStoreComponent("europe", "parsec", "john",
"secret")
# Ok
client_mock().head_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "404"
}}, operation_name="HEAD")
await blockstore.create("org42", 123, "content")
client_mock().put_object.assert_called_with(Bucket="parsec",
Key="org42/123",
Body="content")
client_mock().put_object.reset_mock()
# Already exist
client_mock().head_object.side_effect = None
with pytest.raises(BlockAlreadyExistsError):
await blockstore.create("org42", 123, "content")
client_mock().put_object.assert_not_called()
# Connection error at HEAD
client_mock().head_object.side_effect = S3EndpointConnectionError(
endpoint_url="url")
with pytest.raises(BlockTimeoutError):
await blockstore.create("org42", 123, "content")
client_mock().put_object.assert_not_called()
# Unknown exception at HEAD
client_mock().head_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "401"
}}, operation_name="HEAD")
with pytest.raises(BlockTimeoutError):
await blockstore.create("org42", 123, "content")
# Connection error at PUT
client_mock().head_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "404"
}}, operation_name="HEAD")
client_mock().put_object.side_effect = S3EndpointConnectionError(
endpoint_url="url")
with pytest.raises(BlockTimeoutError):
await blockstore.create("org42", 123, "content")
# Unknown exception at PUT
client_mock().put_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "401"
}}, operation_name="PUT")
with pytest.raises(BlockTimeoutError):
await blockstore.create("org42", 123, "content")
async def test_perform_block_read_not_found(self, s3_block_connection):
s3_block_connection.mocked_boto3_client.get_object.side_effect = \
S3ClientError({'Error': {}}, 'get_object')
with pytest.raises(BlockNotFound):
await s3_block_connection.read('unknown_id')
async def test_block_create_duplicate_id(self, s3_block_connection):
s3_block_connection.mocked_boto3_client.put_object.side_effect = \
S3ClientError({'Error': {}}, 'put_object')
with pytest.raises(BlockError):
await s3_block_connection.create('42', b'foo')