def to_delete(request: Any,
manager: storage.StorageManager) -> List[Dict[str, Any]]:
metadata = []
for _ in range(request.param):
with manager.store_path() as (storage_id, path):
storage_util.create_checkpoint(path)
metadata.append(
storage.StorageMetadata(storage_id,
manager._list_directory(path)))
assert len(os.listdir(manager._base_path)) == request.param
return [simplejson.loads(util.json_encode(m)) for m in metadata]
def test_list_directory() -> None:
root = os.path.join(os.path.dirname(__file__), "fixtures")
assert set(StorageManager._list_directory(root)) == {
"root.txt",
"nested/",
"nested/nested.txt",
"nested/another.txt",
}
def delete_checkpoints(manager: storage.StorageManager,
to_delete: List[Dict[str, Any]], dry_run: bool) -> None:
"""
Delete some of the checkpoints associated with a single
experiment. `to_delete` is a list of two-element dicts,
{"uuid": str, "resources": List[str]}.
"""
logging.info("Deleting {} checkpoints".format(len(to_delete)))
for record in to_delete:
metadata = storage.StorageMetadata.from_json(record)
if not dry_run:
logging.info("Deleting checkpoint {}".format(metadata))
manager.delete(metadata)
else:
logging.info("Dry run: deleting checkpoint {}".format(
metadata.storage_id))
logging.info("Finished deleting {} checkpoints".format(len(to_delete)))
def maybe_load_checkpoint(
storage_mgr: storage.StorageManager, checkpoint: Optional[Dict[str, Any]]
) -> Iterator[Optional[pathlib.Path]]:
"""
Either wrap a storage_mgr.restore_path() context manager, or be a noop
context manager if there is no checkpoint to load.
"""
if checkpoint is None:
yield None
else:
metadata = storage.StorageMetadata.from_json(checkpoint)
logging.info("Restoring trial from checkpoint {}".format(metadata.storage_id))
with storage_mgr.restore_path(metadata) as path:
yield pathlib.Path(path)
def test_list_nonexistent_directory() -> None:
root = "./non-existent-directory"
assert not os.path.exists(root)
with pytest.raises(CheckFailedError, match="must be an extant directory"):
StorageManager._list_directory(root)
def test_list_directory_on_file() -> None:
root = os.path.join(os.path.dirname(__file__), "fixtures", "root.txt")
assert os.path.exists(root)
with pytest.raises(CheckFailedError, match="must be an extant directory"):
StorageManager._list_directory(root)
