def to_delete(request: Any,
manager: storage.StorageManager) -> List[Dict[str, Any]]:
metadata = []
for _ in range(request.param):
with manager.store_path() as (storage_id, path):
storage_util.create_checkpoint(path)
metadata.append(
storage.StorageMetadata(storage_id,
manager._list_directory(path)))
assert len(os.listdir(manager._base_path)) == request.param
return [simplejson.loads(util.json_encode(m)) for m in metadata]
def delete_checkpoints(manager: storage.StorageManager, to_delete: List[str],
dry_run: bool) -> None:
"""
Delete some of the checkpoints associated with a single experiment.
"""
logging.info("Deleting {} checkpoints".format(len(to_delete)))
for storage_id in to_delete:
if not dry_run:
logging.info(f"Deleting checkpoint {storage_id}")
manager.delete(storage_id)
else:
logging.info(f"Dry run: deleting checkpoint {storage_id}")
def delete_checkpoints(manager: storage.StorageManager,
to_delete: List[Dict[str, Any]], dry_run: bool) -> None:
"""
Delete some of the checkpoints associated with a single
experiment. `to_delete` is a list of two-element dicts,
{"uuid": str, "resources": List[str]}.
"""
logging.info("Deleting {} checkpoints".format(len(to_delete)))
for record in to_delete:
metadata = storage.StorageMetadata.from_json(record)
if not dry_run:
logging.info("Deleting checkpoint {}".format(metadata))
manager.delete(metadata)
else:
logging.info("Dry run: deleting checkpoint {}".format(
metadata.storage_id))
def test_list_directory() -> None:
root = os.path.join(os.path.dirname(__file__), "fixtures")
assert set(StorageManager._list_directory(root)) == {
"root.txt",
"nested/",
"nested/nested.txt",
"nested/another.txt",
}
def to_delete(request: Any, manager: storage.StorageManager) -> List[str]:
storage_ids = []
for _ in range(request.param):
storage_id = str(uuid.uuid4())
with manager.store_path(storage_id) as path:
storage_util.create_checkpoint(path)
storage_ids.append(storage_id)
assert len(os.listdir(manager._base_path)) == request.param
return storage_ids
def maybe_load_checkpoint(
storage_mgr: storage.StorageManager,
checkpoint: Optional[Dict[str,
Any]]) -> Iterator[Optional[pathlib.Path]]:
"""
Either wrap a storage_mgr.restore_path() context manager, or be a noop
context manager if there is no checkpoint to load.
"""
if checkpoint is None:
yield None
else:
metadata = storage.StorageMetadata.from_json(checkpoint)
logging.info("Restoring trial from checkpoint {}".format(
metadata.storage_id))
with storage_mgr.restore_path(metadata) as path:
yield pathlib.Path(path)
def test_list_nonexistent_directory() -> None:
root = "./non-existent-directory"
assert not os.path.exists(root)
with pytest.raises(FileNotFoundError, match=root):
StorageManager._list_directory(root)
def test_list_directory_on_file() -> None:
root = os.path.join(os.path.dirname(__file__), "fixtures", "root.txt")
assert os.path.exists(root)
with pytest.raises(NotADirectoryError, match=root):
StorageManager._list_directory(root)
def test_list_nonexistent_directory() -> None:
root = "./non-existent-directory"
assert not os.path.exists(root)
with pytest.raises(CheckFailedError, match="must be an extant directory"):
StorageManager._list_directory(root)
def test_list_directory_on_file() -> None:
root = os.path.join(os.path.dirname(__file__), "fixtures", "root.txt")
assert os.path.exists(root)
with pytest.raises(CheckFailedError, match="must be an extant directory"):
StorageManager._list_directory(root)
def run_storage_lifecycle_test(
manager: storage.StorageManager,
post_delete_cb: Optional[Callable] = None,
) -> None:
checkpoints = []
for _ in range(2):
storage_id = str(uuid.uuid4())
with manager.store_path(storage_id) as path:
create_checkpoint(path)
checkpoints.append(storage_id)
for storage_id in checkpoints:
# Load checkpoint.
with manager.restore_path(storage_id) as path:
validate_checkpoint(path)
# Delete.
manager.delete(storage_id)
# Ensure it is gone.
with pytest.raises(errors.CheckpointNotFound):
with manager.restore_path(storage_id) as path:
pass
# Allow for backend-specific inspection.
if post_delete_cb is not None:
post_delete_cb(storage_id)
# Again, using upload/download instead of store_path/restore_path.
checkpoints = []
for _ in range(2):
storage_id = str(uuid.uuid4())
path = pathlib.Path(f"/tmp/storage_lifecycle_test-{storage_id}")
try:
create_checkpoint(path)
manager.upload(path, storage_id)
checkpoints.append(storage_id)
finally:
shutil.rmtree(path, ignore_errors=True)
for storage_id in checkpoints:
path = pathlib.Path(f"/tmp/storage_lifecycle_test-{storage_id}")
try:
manager.download(storage_id, path)
validate_checkpoint(path)
finally:
shutil.rmtree(path, ignore_errors=True)
manager.delete(storage_id)
with pytest.raises(errors.CheckpointNotFound):
manager.download(storage_id, path)
if post_delete_cb is not None:
post_delete_cb(storage_id)