async def run(
cls,
device: LocalDevice,
path: Path,
workspace_id: EntryID,
cache_size: int = DEFAULT_BLOCK_CACHE_SIZE,
vacuum_threshold: int = DEFAULT_CHUNK_VACUUM_THRESHOLD,
) -> AsyncIterator["WorkspaceStorage"]:
data_path = path / WORKSPACE_DATA_STORAGE_NAME
cache_path = path / WORKSPACE_CACHE_STORAGE_NAME
# Local cache storage service
async with LocalDatabase.run(cache_path) as cache_localdb:
# Local data storage service
async with LocalDatabase.run(
data_path,
vacuum_threshold=vacuum_threshold) as data_localdb:
# Block storage service
async with BlockStorage.run(
device, cache_localdb,
cache_size=cache_size) as block_storage:
# Manifest storage service
async with ManifestStorage.run(
device, data_localdb,
workspace_id) as manifest_storage:
# Chunk storage service
async with ChunkStorage.run(
device, data_localdb) as chunk_storage:
# Instanciate workspace storage
instance = cls(
device,
path,
workspace_id,
data_localdb=data_localdb,
cache_localdb=cache_localdb,
block_storage=block_storage,
chunk_storage=chunk_storage,
manifest_storage=manifest_storage,
)
# Load "prevent sync" pattern
await instance._load_prevent_sync_pattern()
# Yield point
yield instance
async def run(
cls,
device: LocalDevice,
path: Path,
workspace_id: EntryID,
cache_size=DEFAULT_BLOCK_CACHE_SIZE,
vacuum_threshold=DEFAULT_CHUNK_VACUUM_THRESHOLD,
):
data_path = path / WORKSPACE_DATA_STORAGE_NAME
cache_path = path / WORKSPACE_CACHE_STORAGE_NAME
# Local cache storage service
async with LocalDatabase.run(cache_path) as cache_localdb:
# Local data storage service
async with LocalDatabase.run(
data_path,
vacuum_threshold=vacuum_threshold) as data_localdb:
# Block storage service
async with BlockStorage.run(
device, cache_localdb,
cache_size=cache_size) as block_storage:
# Manifest storage service
async with ManifestStorage.run(
device, data_localdb,
workspace_id) as manifest_storage:
# Chunk storage service
async with ChunkStorage.run(
device, data_localdb) as chunk_storage:
# Instanciate workspace storage
yield cls(
device,
path,
workspace_id,
data_localdb=data_localdb,
cache_localdb=cache_localdb,
block_storage=block_storage,
chunk_storage=chunk_storage,
manifest_storage=manifest_storage,
)
async def workspace_storage_non_speculative_init(
data_base_dir: Path, device: LocalDevice,
workspace_id: EntryID) -> None:
db_path = get_workspace_data_storage_db_path(data_base_dir, device,
workspace_id)
# Local data storage service
async with LocalDatabase.run(db_path) as data_localdb:
# Manifest storage service
async with ManifestStorage.run(device, data_localdb,
workspace_id) as manifest_storage:
timestamp = device.timestamp()
manifest = LocalWorkspaceManifest.new_placeholder(
author=device.device_id,
id=workspace_id,
timestamp=timestamp,
speculative=False)
await manifest_storage.set_manifest(workspace_id, manifest)
async def run(cls, device: LocalDevice, path: Path):
# Local database service
async with LocalDatabase.run(path / USER_STORAGE_NAME) as localdb:
# Manifest storage service
async with ManifestStorage.run(
device, localdb,
device.user_manifest_id) as manifest_storage:
# Instanciate the user storage
self = cls(device, path, device.user_manifest_id,
manifest_storage)
# Populate the cache with the user manifest to be able to
# access it synchronously at all time
await self._load_user_manifest()
assert self.user_manifest_id in self.manifest_storage._cache
yield self
async def user_storage_non_speculative_init(data_base_dir: Path,
device: LocalDevice) -> None:
data_path = get_user_data_storage_db_path(data_base_dir, device)
# Local data storage service
async with LocalDatabase.run(data_path) as localdb:
# Manifest storage service
async with ManifestStorage.run(
device, localdb, device.user_manifest_id) as manifest_storage:
timestamp = device.timestamp()
manifest = LocalUserManifest.new_placeholder(
author=device.device_id,
id=device.user_manifest_id,
timestamp=timestamp,
speculative=False,
)
await manifest_storage.set_manifest(device.user_manifest_id,
manifest)
async def run(cls, data_base_dir: Path,
device: LocalDevice) -> AsyncIterator["UserStorage"]:
data_path = get_user_data_storage_db_path(data_base_dir, device)
# Local database service
async with LocalDatabase.run(data_path) as localdb:
# Manifest storage service
async with ManifestStorage.run(
device, localdb,
device.user_manifest_id) as manifest_storage:
# Instanciate the user storage
self = cls(device, device.user_manifest_id, manifest_storage)
# Populate the cache with the user manifest to be able to
# access it synchronously at all time
await self._load_user_manifest()
assert self.user_manifest_id in self.manifest_storage._cache
yield self
async def test_workspace_manifest_access_while_speculative(
user_fs_factory, alice):
# Speculative workspace occurs when workspace is shared to a new user, or
# when a device gets it local data removed. We use the latter here (even if
# it is the less likely of the two) given it is simpler to do in the test.
with freeze_time("2000-01-01"):
async with user_fs_factory(alice) as user_fs:
wksp_id = await user_fs.workspace_create(EntryName("wksp"))
# Retreive where the database is stored
wksp = user_fs.get_workspace(wksp_id)
wksp_manifest_storage_db_path = wksp.local_storage.manifest_storage.path
# Manually drop the workspace manifest from database
async with LocalDatabase.run(
path=wksp_manifest_storage_db_path) as localdb:
async with ManifestStorage.run(device=alice,
localdb=localdb,
realm_id=wksp_id) as manifest_storage:
await manifest_storage.clear_manifest(wksp_id)
# Now re-start the userfs (the same local storage will be used)
with freeze_time("2000-01-02"):
async with user_fs_factory(alice) as user_fs:
with freeze_time("2000-01-03"):
wksp = user_fs.get_workspace(wksp_id)
root_stat = await wksp.path_info("/")
assert root_stat == {
"id": wksp_id,
"base_version": 0,
"created": datetime(2000, 1, 2),
"updated": datetime(2000, 1, 2),
"is_placeholder": True,
"need_sync": True,
"type": "folder",
"children": [],
"confinement_point": None,
}
async def run(
cls,
data_base_dir: Path,
device: LocalDevice,
workspace_id: EntryID,
cache_size: int = DEFAULT_WORKSPACE_STORAGE_CACHE_SIZE,
data_vacuum_threshold: int = DEFAULT_CHUNK_VACUUM_THRESHOLD,
) -> AsyncIterator["WorkspaceStorage"]:
data_path = get_workspace_data_storage_db_path(data_base_dir, device,
workspace_id)
cache_path = get_workspace_cache_storage_db_path(
data_base_dir, device, workspace_id)
# The cache database usually doesn't require vacuuming as it already has a maximum size.
# However, vacuuming might still be necessary after a change in the configuration.
# The cache size plus 10% seems like a reasonable configuration to avoid false positive.
cache_localdb_vaccuum_threshold = int(cache_size * 1.1)
# Local cache storage service
async with LocalDatabase.run(
cache_path,
vacuum_threshold=cache_localdb_vaccuum_threshold) as cache_localdb:
# Local data storage service
async with LocalDatabase.run(
data_path,
vacuum_threshold=data_vacuum_threshold) as data_localdb:
# Block storage service
async with BlockStorage.run(
device, cache_localdb,
cache_size=cache_size) as block_storage:
# Clean up block storage and run vacuum if necessary
# (e.g after changing the cache size)
await block_storage.cleanup()
await cache_localdb.run_vacuum()
# Manifest storage service
async with ManifestStorage.run(
device, data_localdb,
workspace_id) as manifest_storage:
# Chunk storage service
async with ChunkStorage.run(
device, data_localdb) as chunk_storage:
# Instanciate workspace storage
instance = cls(
device,
workspace_id,
data_localdb=data_localdb,
cache_localdb=cache_localdb,
block_storage=block_storage,
chunk_storage=chunk_storage,
manifest_storage=manifest_storage,
)
# Populate the cache with the workspace manifest to be able to
# access it synchronously at all time
await instance._load_workspace_manifest()
assert instance.workspace_id in instance.manifest_storage._cache
# Load "prevent sync" pattern
await instance._load_prevent_sync_pattern()
# Yield point
yield instance