async def test_block_create_and_read(alice_backend_sock, realm):
await block_create(alice_backend_sock, BLOCK_ID, realm, BLOCK_DATA)
rep = await block_read(alice_backend_sock, BLOCK_ID)
assert rep == {"status": "ok", "block": BLOCK_DATA}
# Test not found as well
dummy_id = BlockID.from_hex("00000000000000000000000000000002")
rep = await block_read(alice_backend_sock, dummy_id)
assert rep == {"status": "not_found"}
async def test_s3_read(caplog):
org_id = OrganizationID("org42")
block_id = BlockID.from_hex("0694a21176354e8295e28a543e5887f9")
def _assert_log():
log = caplog.assert_occured_once("[warning ] Block read error")
assert f"organization_id={org_id}" in log
assert f"block_id={block_id}" in log
assert len(caplog.messages) == 1
caplog.clear()
with mock.patch("boto3.client") as client_mock:
client_mock.return_value = Mock()
client_mock().head_bucket.return_value = True
blockstore = S3BlockStoreComponent("europe", "parsec", "john",
"secret")
# Ok
response_mock = Mock()
response_mock.read.return_value = "content"
client_mock().get_object.return_value = {"Body": response_mock}
assert await blockstore.read(org_id, block_id) == "content"
client_mock().get_object.assert_called_once_with(
Bucket="parsec", Key="org42/0694a211-7635-4e82-95e2-8a543e5887f9")
client_mock().get_object.reset_mock()
assert not caplog.messages
# Not found
client_mock().get_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "404"
}}, operation_name="GET")
with pytest.raises(BlockStoreError):
assert await blockstore.read(org_id, block_id)
_assert_log()
# Connection error
client_mock().get_object.side_effect = S3EndpointConnectionError(
endpoint_url="url")
with pytest.raises(BlockStoreError):
assert await blockstore.read(org_id, block_id)
_assert_log()
# Unknown exception
client_mock().get_object.side_effect = S3ClientError(
error_response={"Error": {
"Code": "401"
}}, operation_name="GET")
with pytest.raises(BlockStoreError):
assert await blockstore.read(org_id, block_id)
_assert_log()
async def test_realm_stats_ok(alice_backend_sock, realm):
# Create new data
await block_create(alice_backend_sock,
realm_id=realm,
block_id=BlockID.new(),
block=b"1234")
rep = await realm_stats(alice_backend_sock, realm_id=realm)
assert rep == {"status": "ok", "blocks_size": 4, "vlobs_size": 0}
# Create new metadata
await vlob_create(alice_backend_sock,
realm_id=realm,
vlob_id=VlobID.new(),
blob=b"1234")
rep = await realm_stats(alice_backend_sock, realm_id=realm)
assert rep == {"status": "ok", "blocks_size": 4, "vlobs_size": 4}
async def test_swift_create(caplog):
org_id = OrganizationID("org42")
block_id = BlockID.from_hex("0694a21176354e8295e28a543e5887f9")
def _assert_log():
log = caplog.assert_occured_once("[warning ] Block create error")
assert f"organization_id={org_id}" in log
assert f"block_id={block_id}" in log
assert len(caplog.messages) == 1
caplog.clear()
with mock.patch("swiftclient.Connection") as connection_mock:
connection_mock.return_value = Mock()
connection_mock().head_container.return_value = True
blockstore = SwiftBlockStoreComponent("http://url", "scille", "parsec",
"john", "secret")
# Ok
connection_mock().get_object.side_effect = ClientException(
http_status=404, msg="")
await blockstore.create(org_id, block_id, "content")
connection_mock().put_object.assert_called_with(
"parsec", "org42/0694a211-7635-4e82-95e2-8a543e5887f9", "content")
connection_mock().put_object.reset_mock()
assert not caplog.messages
# Connection error at PUT
connection_mock().get_object.side_effect = ClientException(
msg="Connection error")
connection_mock().put_object.side_effect = ClientException(
msg="Connection error")
with pytest.raises(BlockStoreError):
await blockstore.create(org_id, block_id, "content")
_assert_log()
# Unknown exception at PUT
connection_mock().put_object.side_effect = ClientException(
http_status=500, msg="")
with pytest.raises(BlockStoreError):
await blockstore.create(org_id, block_id, "content")
_assert_log()
async def test_organization_stats_data(alice_backend_sock, realm,
realm_factory, alice, backend):
stats = await organization_stats(alice_backend_sock)
assert stats == {
"status":
"ok",
"data_size":
0,
"metadata_size":
ANY,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
4,
}
initial_metadata_size = stats["metadata_size"]
# Create new metadata
await backend.vlob.create(
organization_id=alice.organization_id,
author=alice.device_id,
realm_id=realm,
encryption_revision=1,
vlob_id=VlobID.new(),
timestamp=pendulum.now(),
blob=b"1234",
)
stats = await organization_stats(alice_backend_sock)
assert stats == {
"status":
"ok",
"data_size":
0,
"metadata_size":
initial_metadata_size + 4,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
4,
}
# Create new data
await backend.block.create(
organization_id=alice.organization_id,
author=alice.device_id,
block_id=BlockID.new(),
realm_id=realm,
block=b"1234",
)
stats = await organization_stats(alice_backend_sock)
assert stats == {
"status":
"ok",
"data_size":
4,
"metadata_size":
initial_metadata_size + 4,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
4,
}
# create new workspace
await realm_factory(backend, alice)
stats = await organization_stats(alice_backend_sock)
assert stats == {
"status":
"ok",
"data_size":
4,
"metadata_size":
initial_metadata_size + 4,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
5,
}
async def test_block_create_check_access_rights(backend, alice, bob,
bob_backend_sock, realm,
next_timestamp):
block_id = BlockID.new()
# User not part of the realm
rep = await block_create(bob_backend_sock,
block_id,
realm,
BLOCK_DATA,
check_rep=False)
assert rep == {"status": "not_allowed"}
# User part of the realm with various role
for role, access_granted in [
(RealmRole.READER, False),
(RealmRole.CONTRIBUTOR, True),
(RealmRole.MANAGER, True),
(RealmRole.OWNER, True),
]:
await backend.realm.update_roles(
alice.organization_id,
RealmGrantedRole(
certificate=b"<dummy>",
realm_id=realm,
user_id=bob.user_id,
role=role,
granted_by=alice.device_id,
granted_on=next_timestamp(),
),
)
block_id = BlockID.new()
rep = await block_create(bob_backend_sock,
block_id,
realm,
BLOCK_DATA,
check_rep=False)
if access_granted:
assert rep == {"status": "ok"}
else:
assert rep == {"status": "not_allowed"}
# Ensure user that used to be part of the realm have no longer access
await backend.realm.update_roles(
alice.organization_id,
RealmGrantedRole(
certificate=b"<dummy>",
realm_id=realm,
user_id=bob.user_id,
role=None,
granted_by=alice.device_id,
granted_on=next_timestamp(),
),
)
rep = await block_create(bob_backend_sock,
block_id,
realm,
BLOCK_DATA,
check_rep=False)
assert rep == {"status": "not_allowed"}
async def block(backend, alice, realm):
block_id = BlockID.from_hex("0000000000000000000000000000000C")
await backend.block.create(alice.organization_id, alice.device_id,
block_id, realm, BLOCK_DATA)
return block_id
generate_checksum_chunk,
rebuild_block_from_chunks,
)
from parsec.api.protocol import (
BlockID,
VlobID,
block_create_serializer,
block_read_serializer,
packb,
RealmRole,
)
from tests.common import customize_fixtures
from tests.backend.common import block_create, block_read
BLOCK_ID = BlockID.from_hex("00000000000000000000000000000001")
VLOB_ID = VlobID.from_hex("00000000000000000000000000000002")
BLOCK_DATA = b"Hodi ho !"
@pytest.fixture
async def block(backend, alice, realm):
block_id = BlockID.from_hex("0000000000000000000000000000000C")
await backend.block.create(alice.organization_id, alice.device_id,
block_id, realm, BLOCK_DATA)
return block_id
@pytest.mark.trio
async def test_block_read_check_access_rights(backend, alice, bob,
async def test_organization_stats_data(backend_rest_send, realm, realm_factory,
alice, backend):
async def organization_stats():
status, _, body = await backend_rest_send(
f"/administration/organizations/{alice.organization_id}/stats")
assert status == (200, "OK")
return organization_stats_rep_serializer.load(body)
rep = await organization_stats()
assert rep == {
"data_size":
0,
"metadata_size":
ANY,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
4,
}
initial_metadata_size = rep["metadata_size"]
# Create new metadata
await backend.vlob.create(
organization_id=alice.organization_id,
author=alice.device_id,
realm_id=realm,
encryption_revision=1,
vlob_id=VlobID.new(),
timestamp=pendulum.now(),
blob=b"1234",
)
rep = await organization_stats()
assert rep == {
"data_size":
0,
"metadata_size":
initial_metadata_size + 4,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
4,
}
# Create new data
await backend.block.create(
organization_id=alice.organization_id,
author=alice.device_id,
block_id=BlockID.new(),
realm_id=realm,
block=b"1234",
)
rep = await organization_stats()
assert rep == {
"data_size":
4,
"metadata_size":
initial_metadata_size + 4,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
4,
}
# create new workspace
await realm_factory(backend, alice)
rep = await organization_stats()
assert rep == {
"data_size":
4,
"metadata_size":
initial_metadata_size + 4,
"users":
3,
"active_users":
3,
"users_per_profile_detail": [
{
"profile": UserProfile.ADMIN,
"active": 2,
"revoked": 0
},
{
"profile": UserProfile.STANDARD,
"active": 1,
"revoked": 0
},
{
"profile": UserProfile.OUTSIDER,
"active": 0,
"revoked": 0
},
],
"realms":
5,
}
async def test_access_during_reencryption(
backend, alice_backend_sock, alice, realm_factory, next_timestamp
):
# First initialize a nice realm with block and vlob
realm_id = await realm_factory(backend, author=alice)
vlob_id = VlobID.new()
block_id = BlockID.new()
await backend.vlob.create(
organization_id=alice.organization_id,
author=alice.device_id,
realm_id=realm_id,
encryption_revision=1,
vlob_id=vlob_id,
timestamp=next_timestamp(),
blob=b"v1",
)
await backend.block.create(
organization_id=alice.organization_id,
author=alice.device_id,
realm_id=realm_id,
block_id=block_id,
block=b"<block_data>",
)
async def _assert_write_access_disallowed(encryption_revision):
rep = await vlob_create(
alice_backend_sock,
realm_id=realm_id,
vlob_id=VlobID.new(),
blob=b"data",
encryption_revision=encryption_revision,
check_rep=False,
)
assert rep == {"status": "in_maintenance"}
rep = await vlob_update(
alice_backend_sock,
vlob_id,
version=2,
blob=b"data",
encryption_revision=encryption_revision,
check_rep=False,
)
assert rep == {"status": "in_maintenance"}
rep = await block_create(
alice_backend_sock, block_id=block_id, realm_id=realm_id, block=b"data", check_rep=False
)
assert rep == {"status": "in_maintenance"}
async def _assert_read_access_allowed(encryption_revision, expected_blob=b"v1"):
rep = await vlob_read(
alice_backend_sock, vlob_id=vlob_id, version=1, encryption_revision=encryption_revision
)
assert rep["status"] == "ok"
assert rep["blob"] == expected_blob
rep = await block_read(alice_backend_sock, block_id=block_id)
assert rep == {"status": "ok", "block": b"<block_data>"}
# For good measure, also try those read-only commands even if they
# are encryption-revision agnostic
rep = await vlob_list_versions(alice_backend_sock, vlob_id=vlob_id)
assert rep["status"] == "ok"
rep = await vlob_poll_changes(alice_backend_sock, realm_id=realm_id, last_checkpoint=0)
assert rep["status"] == "ok"
async def _assert_read_access_bad_encryption_revision(encryption_revision, expected_status):
rep = await vlob_read(
alice_backend_sock, vlob_id=vlob_id, version=1, encryption_revision=encryption_revision
)
assert rep == {"status": expected_status}
# Sanity check just to make we can access the data with initial encryption revision
await _assert_read_access_allowed(1)
# Now start reencryption
await backend.realm.start_reencryption_maintenance(
organization_id=alice.organization_id,
author=alice.device_id,
realm_id=realm_id,
encryption_revision=2,
per_participant_message={alice.user_id: b"<whatever>"},
timestamp=pendulum_now(),
)
# Only read with old encryption revision is now allowed
await _assert_read_access_allowed(1)
await _assert_read_access_bad_encryption_revision(2, expected_status="in_maintenance")
await _assert_write_access_disallowed(1)
await _assert_write_access_disallowed(2)
# Actually reencrypt the vlob data, this shouldn't affect us for the moment
# given reencryption is not formally finished
await backend.vlob.maintenance_save_reencryption_batch(
organization_id=alice.organization_id,
author=alice.device_id,
realm_id=realm_id,
encryption_revision=2,
batch=[(vlob_id, 1, b"v2")],
)
await _assert_read_access_allowed(1)
await _assert_read_access_bad_encryption_revision(2, expected_status="in_maintenance")
await _assert_write_access_disallowed(1)
await _assert_write_access_disallowed(2)
# Finish the reencryption
await backend.realm.finish_reencryption_maintenance(
organization_id=alice.organization_id,
author=alice.device_id,
realm_id=realm_id,
encryption_revision=2,
)
# Now only the new encryption revision is allowed
await _assert_read_access_allowed(2, expected_blob=b"v2")
await _assert_read_access_bad_encryption_revision(1, expected_status="bad_encryption_revision")