def test_local_filesystem():
local_fs1 = LocalFileSystem.get_instance()
local_fs2 = LocalFileSystem.get_instance()
assert local_fs1 is local_fs2
with tempfile.TemporaryDirectory() as tempdir:
file_path = os.path.join(tempdir, 'test')
with open(file_path, 'wb') as f:
f.write(b'text for test')
assert local_fs1.stat(tempdir)['type'] == 'directory'
assert local_fs1.stat(file_path)['type'] == 'file'
assert len(glob(tempdir + '*')) == 1
async def storage_context(ray_start_regular, request):
if request.param == 'filesystem':
tempdir = tempfile.mkdtemp()
params, teardown_params = await FileSystemStorage.setup(
fs=LocalFileSystem(),
root_dirs=[tempdir],
level=StorageLevel.DISK)
storage = FileSystemStorage(**params)
assert storage.level == StorageLevel.DISK
yield storage
await storage.teardown(**teardown_params)
elif request.param == 'plasma':
plasma_storage_size = 10 * 1024 * 1024
if sys.platform == 'darwin':
plasma_dir = '/tmp'
else:
plasma_dir = '/dev/shm'
params, teardown_params = await PlasmaStorage.setup(
store_memory=plasma_storage_size,
plasma_directory=plasma_dir,
check_dir_size=False)
storage = PlasmaStorage(**params)
assert storage.level == StorageLevel.MEMORY
yield storage
await PlasmaStorage.teardown(**teardown_params)
elif request.param == 'vineyard':
vineyard_size = '256M'
vineyard_socket = '/tmp/vineyard.sock'
params, teardown_params = await VineyardStorage.setup(
vineyard_size=vineyard_size,
vineyard_socket=vineyard_socket)
storage = VineyardStorage(**params)
assert storage.level == StorageLevel.MEMORY
yield storage
await VineyardStorage.teardown(**teardown_params)
elif request.param == 'shared_memory':
params, teardown_params = await SharedMemoryStorage.setup()
storage = SharedMemoryStorage(**params)
assert storage.level == StorageLevel.MEMORY
yield storage
teardown_params['object_ids'] = storage._object_ids
await SharedMemoryStorage.teardown(**teardown_params)
elif request.param == 'ray':
params, teardown_params = await RayStorage.setup()
storage = RayStorage(**params)
assert storage.level == StorageLevel.MEMORY | StorageLevel.REMOTE
yield storage
await RayStorage.teardown(**teardown_params)
async def test_aio_filesystem():
local_fs = LocalFileSystem.get_instance()
aio_fs = AioFilesystem(local_fs)
assert aio_fs.pathsep == local_fs.pathsep
with tempfile.TemporaryDirectory() as tempdir:
file_path = os.path.join(tempdir, 'test')
with open(file_path, 'wb') as f:
f.write(b'text for test')
stat = await aio_fs.stat(tempdir)
assert stat['type'] == 'directory'
def test_fsmap():
fs = LocalFileSystem.get_instance()
with tempfile.TemporaryDirectory() as root:
fs_map = FSMap(root, fs, check=True)
path = '/to/path/test_file'
test_content = b'text for test'
fs_map[path] = test_content
assert fs_map[path] == test_content
assert len(fs_map) == 1
assert path in fs_map
path2 = '/to/path2/test_file2'
fs_map[path2] = test_content
assert len(fs_map) == 2
del fs_map[path]
assert list(fs_map) == ['to/path2/test_file2']
path3 = '/to2/path3/test_file3'
fs_map[path3] = test_content
assert fs_map.pop(path3) == test_content
assert fs_map.pop(path3, 'fake_content') == 'fake_content'
with pytest.raises(KeyError):
fs_map.pop('not_exist')
fs_map.clear()
assert len(fs_map) == 0
# test root not exist
with pytest.raises(ValueError):
_ = FSMap(root + '/path2', fs, check=True)
# create root
fs_map = FSMap(root + '/path2', fs, create=True)
assert len(fs_map) == 0