def test_replace(tmp_path: pathlib.Path):
file_path = gpathlib.PosixGPath(os.path.join(tmp_path, 'tfds.py'))
file_path.write_text('tfds')
file_path.replace(os.path.join(tmp_path, 'tfds-dataset.py'))
assert not tmp_path.joinpath('tfds.py').exists()
assert tmp_path.joinpath('tfds-dataset.py').exists()
assert tmp_path.joinpath('tfds-dataset.py').read_text() == 'tfds'
mnist_path = gpathlib.PosixGPath(tmp_path / 'mnist.py')
mnist_path.write_text('mnist')
mnist_path.replace(tmp_path / 'mnist-100.py')
assert not tmp_path.joinpath('mnist.py').exists()
assert tmp_path.joinpath('mnist-100.py').exists()
assert tmp_path.joinpath('mnist-100.py').read_text() == 'mnist'
assert len(list(tmp_path.iterdir())) == 2
assert sorted(gpathlib.PosixGPath(tmp_path).iterdir()) == [
tmp_path / 'mnist-100.py', tmp_path / 'tfds-dataset.py'
]
def test_copy():
src_path = gpathlib.PosixGPath('gs://foo.py')
src_path.write_text('abc')
assert not src_path.parent.joinpath('bar.py').exists()
assert not src_path.parent.joinpath('bar2.py').exists()
src_path.copy('gs://bar.py')
src_path.copy(gpathlib.PosixGPath('gs://bar2.py'))
assert src_path.exists()
assert src_path.parent.joinpath('bar.py').read_text() == 'abc'
assert src_path.parent.joinpath('bar2.py').read_text() == 'abc'
def test_repr_gcs():
path = gpathlib.PosixGPath('gs://bucket/dir')
assert isinstance(path, gpathlib.PosixGPath)
assert repr(path) == f'PosixGPath(\'{_GCS_SCHEME}bucket/dir\')'
assert str(path) == f'{_GCS_SCHEME}bucket/dir'
assert os.fspath(path) == f'{_GCS_SCHEME}bucket/dir'
path = path.parent / 'some/other/file.json'
assert isinstance(path, gpathlib.PosixGPath)
assert os.fspath(path) == f'{_GCS_SCHEME}bucket/some/other/file.json'
path = gpathlib.PosixGPath(path, 'other')
assert isinstance(path, gpathlib.PosixGPath)
assert os.fspath(path) == f'{_GCS_SCHEME}bucket/some/other/file.json/other'
def test_repr_s3():
path = gpathlib.PosixGPath('s3://bucket/dir')
assert isinstance(path, gpathlib.PosixGPath)
assert repr(path) == "PosixGPath('s3://bucket/dir')"
assert str(path) == 's3://bucket/dir'
assert os.fspath(path) == 's3://bucket/dir'
path = path.parent / 'some/other/file.json'
assert isinstance(path, gpathlib.PosixGPath)
assert os.fspath(path) == 's3://bucket/some/other/file.json'
path = gpathlib.PosixGPath(path, 'other')
assert isinstance(path, gpathlib.PosixGPath)
assert os.fspath(path) == 's3://bucket/some/other/file.json/other'
def as_path(path: PathLike) -> ReadWritePath:
"""Create a generic `pathlib.Path`-like abstraction.
Depending on the input (e.g. `gs://`, `github://`, `ResourcePath`,...), the
system (Windows, Linux,...), the function will create the right pathlib-like
abstraction.
Args:
path: Pathlike object.
Returns:
path: The `pathlib.Path`-like abstraction.
"""
is_windows = os.name == 'nt'
if isinstance(path, str):
uri_splits = path.split('://', maxsplit=1)
if len(uri_splits) > 1: # str is URI (e.g. `gs://`, `github://`,...)
# On windows, `PosixGPath` is created for `gs://` paths
return _URI_PREFIXES_TO_CLS[uri_splits[0] + '://'](path) # pytype: disable=bad-return-type
elif is_windows:
return gpath.WindowsGPath(path)
else:
return gpath.PosixGPath(path)
elif isinstance(path, _PATHLIKE_CLS):
return path # Forward resource path, gpath,... as-is # pytype: disable=bad-return-type
elif isinstance(path, os.PathLike): # Other `os.fspath` compatible objects
path_cls = gpath.WindowsGPath if is_windows else gpath.PosixGPath
return path_cls(path)
else:
raise TypeError(f'Invalid path type: {path!r}')
def test_open(gcs_mocked_path: pathlib.Path):
files = [
'foo.py', 'bar.py', 'foo_bar.py', 'dataset.json', 'dataset_info.json',
'readme.md'
]
dataset_path = gpathlib.PosixGPath('gs://bucket/dataset')
dataset_path.mkdir(parents=True)
assert dataset_path.exists()
with pytest.raises(ValueError, match='Only UTF-8 encoding supported.'):
dataset_path.open('w', encoding='latin-1')
# open()
for file in files:
with dataset_path.joinpath(file).open('w') as f:
f.write(' ')
# encoding argument
with dataset_path.joinpath('foo.py').open('r', encoding='UTF-8') as f:
f.read()
# iterdir()
assert len(list(gcs_mocked_path.joinpath('bucket/dataset').iterdir())) == 6
def as_path(path: PathLike) -> ReadWritePath:
"""Create a generic `pathlib.Path`-like abstraction.
Depending on the input (e.g. `gs://` url, `ResourcePath`,...), the
system (Windows, Linux,...), the function will create the right pathlib-like
abstraction.
Args:
path: Pathlike object.
Returns:
path: The `pathlib.Path`-like abstraction.
"""
is_windows = os.name == 'nt'
if isinstance(path, str):
if is_windows and not path.startswith(gpath.URI_PREFIXES):
return gpath.WindowsGPath(path)
else:
return gpath.PosixGPath(
path) # On linux, or for `gs://`, uses `GPath`
elif isinstance(path, tuple(_PATHLIKE_CLS)):
return path # Forward resource path, gpath,... as-is # pytype: disable=bad-return-type
elif isinstance(path, os.PathLike): # Other `os.fspath` compatible objects
path_cls = gpath.WindowsGPath if is_windows else gpath.PosixGPath
return path_cls(path)
else:
raise TypeError(f'Invalid path type: {path!r}')
def test_mkdir(gcs_mocked_path: pathlib.Path):
g_path = gpathlib.PosixGPath('gs://bucket')
assert not g_path.exists()
g_path.mkdir()
assert g_path.exists()
with pytest.raises(FileExistsError, match='already exists'):
g_path.mkdir()
assert gcs_mocked_path.joinpath('bucket').exists()
def test_repr(parts):
path = pathlib.Path(*parts)
gpath = gpathlib.PosixGPath(*parts)
assert isinstance(gpath, gpathlib.PosixGPath)
assert str(gpath) == str(path)
assert os.fspath(gpath) == os.fspath(path)
assert gpath == path
assert str(gpath.resolve()) == str(path.resolve())
assert str(gpath.expanduser()) == str(path.expanduser())
assert isinstance(gpath.resolve(), gpathlib.PosixGPath)
assert isinstance(gpath.expanduser(), gpathlib.PosixGPath)
def test_rename(gcs_mocked_path: pathlib.Path):
src_path = gpathlib.PosixGPath('gs://foo.py')
src_path.write_text(' ')
assert gcs_mocked_path.joinpath('foo.py').exists()
src_path.rename('gs://bar.py')
assert not gcs_mocked_path.joinpath('foo.py').exists()
assert gcs_mocked_path.joinpath('bar.py').exists()
file_name = lambda l: l.name
assert sorted(list(map(file_name,
gcs_mocked_path.iterdir()))) == ['bar.py']
def test_gcs(gcs_mocked_path: pathlib.Path):
# mkdir()
gpath = gpathlib.PosixGPath(f'{_GCS_SCHEME}bucket/dir')
gcs_mocked_path = gcs_mocked_path.joinpath('bucket/dir')
assert not gpath.exists()
gpath.mkdir(parents=True)
# exists()
assert gpath.exists()
assert gcs_mocked_path.exists()
# is_dir()
assert gpath.is_dir()
assert gcs_mocked_path.is_dir()
gpath /= 'some_file.txt'
gcs_mocked_path /= 'some_file.txt'
# touch()
assert not gpath.exists()
gpath.touch()
assert gpath.exists()
assert gcs_mocked_path.exists()
# is_file()
assert gpath.is_file()
assert gcs_mocked_path.is_file()
# iterdir()
gpath = gpath.parent
gcs_mocked_path = gcs_mocked_path.parent
assert list(gpath.iterdir()) == [
gpathlib.PosixGPath('gs://bucket/dir/some_file.txt'),
]
assert isinstance(gpath, gpathlib.PosixGPath)
assert not isinstance(gcs_mocked_path, gpathlib.PosixGPath)
def test_unlink():
path = gpathlib.PosixGPath('gs://bucket')
path.mkdir()
path = path / 'text.txt'
with pytest.raises(FileNotFoundError):
path.unlink()
path.unlink(missing_ok=True) # no-op if missing_ok=True
path.touch() # Path created
assert path.exists()
path.unlink() # Path deleted
assert not path.exists()
def test_touch():
root_path = gpathlib.PosixGPath('gs://bucket/')
root_path.mkdir(parents=True)
assert root_path.exists()
# File don't exists, touch create it
file_path = root_path / 'test.txt'
assert not file_path.exists()
file_path.touch()
assert file_path.exists()
assert file_path.read_text() == '' # File content is empty # pylint: disable=g-explicit-bool-comparison
file_path.write_text('Some content')
file_path.touch() # Should be a no-op
assert file_path.read_text() == 'Some content' # Content still exists
with pytest.raises(FileExistsError):
file_path.touch(exist_ok=False)
def test_windows_encoding():
with mock.patch('os.name', 'nt'):
assert os.name == 'nt'
# On windows, paths should be `WindowsGPath`
path = generic_path.as_path('c:/Program Files/text.txt')
assert isinstance(path, gpath.WindowsGPath)
path = generic_path.as_path(pathlib.PosixPath('some_dir/abc'))
assert isinstance(path, gpath.WindowsGPath)
# Other `GPath` and `gs://` should be `PosixPurePath`
path = generic_path.as_path('gs://some_dir/abc')
assert not isinstance(path, gpath.WindowsGPath)
assert isinstance(path, gpath.PosixGPath)
path = generic_path.as_path(gpath.PosixGPath('some_dir/abc'))
assert not isinstance(path, gpath.WindowsGPath)
assert isinstance(path, gpath.PosixGPath)
def test_read_write(gcs_mocked_path: pathlib.Path): # pylint: disable=unused-argument
gpath = gpathlib.PosixGPath('gs://file.txt')
with gpath.open('w') as f:
f.write('abcd')
with gpath.open('r') as f:
assert f.read() == 'abcd'
gpath.write_text('def')
assert gpath.read_text() == 'def'
with gpath.open('wb') as f:
f.write(b'ghi')
with gpath.open('rb') as f:
assert f.read() == b'ghi'
gpath.write_bytes(b'def')
assert gpath.read_bytes() == b'def'
def test_read_write():
gpath = gpathlib.PosixGPath('gs://file.txt')
with gpath.open('w') as f:
f.write('abcd')
with gpath.open('r') as f:
assert f.read() == 'abcd'
gpath.write_text('def')
assert gpath.read_text() == 'def'
with gpath.open('wb') as f:
f.write(b'ghi')
with gpath.open('rb') as f:
assert f.read() == b'ghi'
gpath.write_bytes(b'def')
assert gpath.read_bytes() == b'def'
def test_format():
template_path = gpathlib.PosixGPath('/home/{user}/foo.py')
template_path = template_path.format(user='******')
assert template_path == gpathlib.PosixGPath('/home/adibou/foo.py')