def test_replace(tmp_path: pathlib.Path):
file_path = gpathlib.GPath(os.path.join(tmp_path, 'tfds.py'))
file_path.write_text('tfds')
file_path.replace(os.path.join(tmp_path, 'tfds-dataset.py'))
assert not tmp_path.joinpath('tfds.py').exists()
assert tmp_path.joinpath('tfds-dataset.py').exists()
assert tmp_path.joinpath('tfds-dataset.py').read_text() == 'tfds'
mnist_path = gpathlib.GPath(tmp_path / 'mnist.py')
mnist_path.write_text('mnist')
mnist_path.replace(tmp_path / 'mnist-100.py')
assert not tmp_path.joinpath('mnist.py').exists()
assert tmp_path.joinpath('mnist-100.py').exists()
assert tmp_path.joinpath('mnist-100.py').read_text() == 'mnist'
assert len(list(tmp_path.iterdir())) == 2
assert sorted(gpathlib.GPath(tmp_path).iterdir()) == [
tmp_path / 'mnist-100.py', tmp_path / 'tfds-dataset.py'
]
def test_repr_gcs():
path = gpathlib.GPath('gs://bucket/dir')
assert isinstance(path, gpathlib.GPath)
assert repr(path) == f'GPath(\'{_GCS_SCHEME}bucket/dir\')'
assert str(path) == f'{_GCS_SCHEME}bucket/dir'
assert os.fspath(path) == f'{_GCS_SCHEME}bucket/dir'
path = path.parent / 'some/other/file.json'
assert isinstance(path, gpathlib.GPath)
assert os.fspath(path) == f'{_GCS_SCHEME}bucket/some/other/file.json'
def test_repr(parts):
path = pathlib.Path(*parts)
gpath = gpathlib.GPath(*parts)
assert isinstance(gpath, gpathlib.GPath)
assert str(gpath) == str(path)
assert os.fspath(gpath) == os.fspath(path)
assert gpath == path
assert str(gpath.resolve()) == str(path.resolve())
assert str(gpath.expanduser()) == str(path.expanduser())
def test_mkdir(gcs_mocked_path: pathlib.Path):
g_path = gpathlib.GPath('gs://bucket')
assert not g_path.exists()
g_path.mkdir()
assert g_path.exists()
with pytest.raises(FileExistsError, match='already exists'):
g_path.mkdir()
assert gcs_mocked_path.joinpath('bucket').exists()
def test_rename(gcs_mocked_path: pathlib.Path):
src_path = gpathlib.GPath('gs://foo.py')
src_path.write_text(' ')
assert gcs_mocked_path.joinpath('foo.py').exists()
src_path.rename('gs://bar.py')
assert not gcs_mocked_path.joinpath('foo.py').exists()
assert gcs_mocked_path.joinpath('bar.py').exists()
file_name = lambda l: l.name
assert sorted(list(map(file_name,
gcs_mocked_path.iterdir()))) == ['bar.py']
def test_gcs(gcs_mocked_path: pathlib.Path):
# mkdir()
gpath = gpathlib.GPath(f'{_GCS_SCHEME}bucket/dir')
gcs_mocked_path = gcs_mocked_path.joinpath('bucket/dir')
assert not gpath.exists()
gpath.mkdir(parents=True)
# exists()
assert gpath.exists()
assert gcs_mocked_path.exists()
# is_dir()
assert gpath.is_dir()
assert gcs_mocked_path.is_dir()
gpath /= 'some_file.txt'
gcs_mocked_path /= 'some_file.txt'
# touch()
assert not gpath.exists()
gpath.touch()
assert gpath.exists()
assert gcs_mocked_path.exists()
# is_file()
assert gpath.is_file()
assert gcs_mocked_path.is_file()
# iterdir()
gpath = gpath.parent
gcs_mocked_path = gcs_mocked_path.parent
assert list(gpath.iterdir()) == [
gpathlib.GPath('gs://bucket/dir/some_file.txt'),
]
assert isinstance(gpath, gpathlib.GPath)
assert not isinstance(gcs_mocked_path, gpathlib.GPath)
def test_open(gcs_mocked_path: pathlib.Path):
files = [
'foo.py', 'bar.py', 'foo_bar.py', 'dataset.json', 'dataset_info.json',
'readme.md'
]
dataset_path = gpathlib.GPath('gs://bucket/dataset')
dataset_path.mkdir(parents=True)
assert dataset_path.exists()
# open()
for file in files:
with dataset_path.joinpath(file).open('w') as f:
f.write(' ')
# iterdir()
assert len(list(gcs_mocked_path.joinpath('bucket/dataset').iterdir())) == 6
def test_read_write(gcs_mocked_path: pathlib.Path): # pylint: disable=unused-argument
gpath = gpathlib.GPath('gs://file.txt')
with gpath.open('w') as f:
f.write('abcd')
with gpath.open('r') as f:
assert f.read() == 'abcd'
gpath.write_text('def')
assert gpath.read_text() == 'def'
with gpath.open('wb') as f:
f.write(b'ghi')
with gpath.open('rb') as f:
assert f.read() == b'ghi'
gpath.write_bytes(b'def')
assert gpath.read_bytes() == b'def'
def as_path(path: PathLike) -> ReadWritePath:
"""Create a generic `pathlib.Path`-like abstraction.
This function
Args:
path: Pathlike object.
Returns:
path: The `pathlib.Path`-like abstraction.
"""
if isinstance(path, str):
if os.name == 'nt' and not path.startswith('gs://'):
# On windows, all path are `pathlib.WindowsPath`, as `gpath.GPath` is
# `PosixPurePath`
return pathlib.Path(path)
else:
return gpath.GPath(path) # On linux, or for `gs://`, uses `GPath`
elif isinstance(path, _PATHLIKE_CLS):
return path # Forward resource path, gpath,... as-is
elif isinstance(path, os.PathLike): # Other `os.fspath` compatible objects
return pathlib.Path(path)
else:
raise TypeError(f'Invalid path type: {path!r}')