def to_delete(request: Any, manager: storage.StorageManager) -> List[Dict[str, Any]]: metadata = [] for _ in range(request.param): with manager.store_path() as (storage_id, path): storage_util.create_checkpoint(path) metadata.append( storage.StorageMetadata(storage_id, manager._list_directory(path))) assert len(os.listdir(manager._base_path)) == request.param return [simplejson.loads(util.json_encode(m)) for m in metadata]
def test_list_directory() -> None: root = os.path.join(os.path.dirname(__file__), "fixtures") assert set(StorageManager._list_directory(root)) == { "root.txt", "nested/", "nested/nested.txt", "nested/another.txt", }
def delete_checkpoints(manager: storage.StorageManager, to_delete: List[Dict[str, Any]], dry_run: bool) -> None: """ Delete some of the checkpoints associated with a single experiment. `to_delete` is a list of two-element dicts, {"uuid": str, "resources": List[str]}. """ logging.info("Deleting {} checkpoints".format(len(to_delete))) for record in to_delete: metadata = storage.StorageMetadata.from_json(record) if not dry_run: logging.info("Deleting checkpoint {}".format(metadata)) manager.delete(metadata) else: logging.info("Dry run: deleting checkpoint {}".format( metadata.storage_id)) logging.info("Finished deleting {} checkpoints".format(len(to_delete)))
def maybe_load_checkpoint( storage_mgr: storage.StorageManager, checkpoint: Optional[Dict[str, Any]] ) -> Iterator[Optional[pathlib.Path]]: """ Either wrap a storage_mgr.restore_path() context manager, or be a noop context manager if there is no checkpoint to load. """ if checkpoint is None: yield None else: metadata = storage.StorageMetadata.from_json(checkpoint) logging.info("Restoring trial from checkpoint {}".format(metadata.storage_id)) with storage_mgr.restore_path(metadata) as path: yield pathlib.Path(path)
def test_list_nonexistent_directory() -> None: root = "./non-existent-directory" assert not os.path.exists(root) with pytest.raises(CheckFailedError, match="must be an extant directory"): StorageManager._list_directory(root)
def test_list_directory_on_file() -> None: root = os.path.join(os.path.dirname(__file__), "fixtures", "root.txt") assert os.path.exists(root) with pytest.raises(CheckFailedError, match="must be an extant directory"): StorageManager._list_directory(root)