def clean(pytestconfig):
token_file = Path("cred.json")
token_file2 = Path("cred2.json")
token = pytestconfig.getoption("token")
token2 = pytestconfig.getoption("token2")
process_token(token, token_file)
process_token(token2, token_file2)
TransparentPath._do_update_cache = True
TransparentPath._do_check = True
before_init()
yield
TransparentPath._do_update_cache = False
TransparentPath._do_check = False
path1 = TransparentPath("chien")
path2 = TransparentPath("chien2")
suffixes = [
"", ".zip", ".txt", ".json", ".csv", ".parquet", ".hdf5", ".xlsx"
]
for suffix in suffixes:
path1.with_suffix(suffix).rm(recursive=True,
ignore_kind=True,
absent="ignore")
path1.with_suffix(suffix).rm(recursive=True,
ignore_kind=True,
absent="ignore")
path2.with_suffix(suffix).rm(recursive=True,
ignore_kind=True,
absent="ignore")
path2.with_suffix(suffix).rm(recursive=True,
ignore_kind=True,
absent="ignore")
TransparentPath._do_update_cache = True
TransparentPath._do_check = True
reinit()
def test_glob(clean, fs_kind, pattern, expected):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
dic = {
"chien": "dir",
"chien/chat": "file",
"chien/cheval": "dir",
"chien/cheval/chouette": "file"
}
root = TransparentPath()
for word in dic:
p = root / word
p.rm(absent="ignore", ignore_kind=True)
for word in dic:
p = root / word
if dic[word] == "file":
p.touch()
else:
p.mkdir()
print(list(TransparentPath("chien").ls()))
content = [
str(p).split("chien/")[1] for p in TransparentPath().glob(pattern)
]
assert content == expected
for word in dic:
p = root / word
p.rm(absent="ignore", ignore_kind=True)
def test_append(clean, fs_kind, to_append):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
assert str(TransparentPath("chien").append(to_append)) == str(
TransparentPath(f"chien{to_append}"))
def test_equal(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien")
p2 = TransparentPath("chien")
assert p1 == p2
def test_gt(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien") / "chat"
p2 = TransparentPath("chien")
assert p1 > p2
def test_truediv(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien")
assert (p1 / "chat") == TransparentPath(cc)
assert (p1 / "/chat") == TransparentPath(cc)
def test_isfile(clean, fs_kind, path1, path2, expected):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath(path1)
p1.touch()
p2 = TransparentPath(path2)
assert p2.is_file() == expected
def test_le(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien") / "chat"
p2 = TransparentPath("chien")
p3 = TransparentPath("chien")
assert p2 <= p1
assert p3 <= p2
def test_itruediv(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien")
p1 /= "chat"
assert p1 == TransparentPath(cc)
p1 = TransparentPath("chien")
p1 /= "/chat"
assert p1 == TransparentPath(cc)
def test_multiproject_1(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
TransparentPath.set_global_fs("gcs", token=os.environ["GOOGLE_APPLICATION_CREDENTIALS_2"])
p1 = TransparentPath("code_tests_sand")
p2 = TransparentPath("code_tests")
p3 = TransparentPath("coucou")
assert "gcs" in p1.fs_kind
assert "gcs" in p2.fs_kind
assert p1.fs_kind != p2.fs_kind
assert p3.fs_kind == "local"
def test_put(clean):
print("test_put")
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
TransparentPath.show_state()
localpath = TransparentPath("chien.txt", fs_kind="local")
remotepath = TransparentPath("chien.txt")
localpath.touch()
localpath.put(remotepath)
assert localpath.is_file()
assert remotepath.is_file()
def test_set_global_fs_then_path_with_gs_failed(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
with pytest.raises(TPValueError):
TransparentPath(f"gs://{bucket + 'chat'}/chien", bucket=bucket)
with pytest.raises(TPNotADirectoryError):
TransparentPath(f"gs://{bucket + 'chat'}/chien")
with pytest.raises(TPValueError):
TransparentPath(f"gs://{bucket}/chien", fs="local")
def init_local_class_then_gcs_path(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("local")
p = TransparentPath("chien", fs="gcs", bucket=bucket)
assert str(p.path) == f"{bucket}/chien"
assert str(p) == f"gs://{bucket}/chien"
assert p.__fspath__() == f"gs://{bucket}/chien"
assert "gcs" in p.fs_kind
assert p.fs == TransparentPath.fss["gcs_sandbox-281209"]
assert not TransparentPath.unset
assert len(TransparentPath.fss) == 2
assert TransparentPath.fs_kind == "local"
assert "gcs_sandbox-281209" in list(
TransparentPath.fss.keys()) and "local" in list(
TransparentPath.fss.keys())
assert isinstance(TransparentPath.fss["gcs_sandbox-281209"],
gcsfs.GCSFileSystem)
assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
reinit()
init("local")
with pytest.raises(ValueError):
TransparentPath(f"gs://{bucket}/chien", bucket=bucket + "chien")
with pytest.raises(ValueError):
TransparentPath(f"gs://{bucket}/chien", fs="local", bucket=bucket)
with pytest.raises(ValueError):
TransparentPath(f"gs://{bucket}/chien")
p = TransparentPath(f"gs://{bucket}/chien", )
assert str(p.path) == f"{bucket}/chien"
assert str(p) == f"gs://{bucket}/chien"
assert p.__fspath__() == f"gs://{bucket}/chien"
assert "gcs" in p.fs_kind
assert p.fs == TransparentPath.fss["gcs_sandbox-281209"]
assert not TransparentPath.unset
assert len(TransparentPath.fss) == 2
assert TransparentPath.fs_kind == "local"
assert "gcs_sandbox-281209" in list(
TransparentPath.fss.keys()) and "local" in list(
TransparentPath.fss.keys())
assert isinstance(TransparentPath.fss["gcs_sandbox-281209"],
gcsfs.GCSFileSystem)
assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
def test_rm(clean, fs_kind, path1, path2, kwargs, expected):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath(path1)
p2 = TransparentPath(path2)
if path1 != "":
p1.touch()
if expected is not None:
with pytest.raises(expected):
p2.rm(**kwargs)
else:
p2.rm(**kwargs)
assert not p2.exists()
def test_cd(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
root = TransparentPath()
(root / "chien").mkdir()
(root / "chien" / "chat").touch()
(root / "chien" / "cheval").mkdir()
(root / "chien" / "cheval" / "chouette").touch()
root.cd("chien")
assert root == TransparentPath("chien")
root.cd("..")
assert root == TransparentPath()
def test_gcs_path_without_set_global_fs_fail(clean, args, kwargs):
if skip_gcs["gcs"]:
print("skipped")
return
with pytest.raises(TPNotADirectoryError):
TransparentPath(*args, **kwargs)
def test_contains(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien") / "chat"
assert "chi" in p1
def __init__(self, path: Union[str, Path, "TransparentPath"], show=False):
self.limits = {}
self.path = path
if type(path) == str:
try:
from transparentpath import TransparentPath
path = TransparentPath(path)
except ImportError:
path = Path(path)
if not path.is_file():
return
if show:
logger.info(f"Found threshold file {path}")
if hasattr(path, "read"):
self.limits = path.read()
else:
with open(path) as opath:
self.limits = load(opath)
if show:
message = "\n".join(
[f"{i}: {self.limits[i]}" for i in self.limits])
logger.info(f"Thresholds are \n{message}")
if "coverage" not in self.limits:
self.limits["coverage"] = {"min": 0.05}
logger.info("Coverage limit was not set. Set to minimum 5%.")
elif "min" not in self.limits["coverage"]:
self.limits["coverage"]["min"] = 0.05
logger.info("Coverage lower limit was not set. Set to 5%.")
elif self.limits["coverage"]["min"] == 0:
logger.warning("Coverage limit is set to 0.")
def test_cp(clean, fs_kind1, fs_kind2):
print("test_cp", fs_kind1, fs_kind2)
if skip_gcs[fs_kind1] or skip_gcs[fs_kind2]:
print("skipped")
return
if fs_kind1 != "local":
init(fs_kind1)
elif fs_kind2 != "local":
init(fs_kind2)
path1 = TransparentPath("chien.txt", fs_kind=fs_kind1)
path2 = TransparentPath("chien2.txt", fs_kind=fs_kind2)
path1.touch()
path1.cp(path2)
assert path1.is_file()
assert path2.is_file()
def test_multipleexistenceerror(clean, fs_kind, excep):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
# noinspection PyTypeChecker
with pytest.raises(excep):
p1 = TransparentPath("chien")
p2 = TransparentPath("chien") / "chat"
p2.touch()
if excep == FileExistsError:
p1.touch()
else:
p1.fs.touch(p1.__fspath__())
TransparentPath("chien").read()
def test_buckets(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
assert "code_tests_sand/" in TransparentPath().buckets
def test_zipfile(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
if fs_kind == "local":
data_path = TransparentPath("tests/data/chien.zip")
else:
local_path = TransparentPath("tests/data/chien.zip", fs_kind="local")
data_path = TransparentPath("chien.zip")
local_path.put(data_path)
zf = zipfile.ZipFile(data_path)
text1 = zf.open("text.txt")
text2 = zf.open("text 2.txt")
assert text1.read() == b'chien\n'
assert text2.read() == b'chat\n'
def test_set_global_fs_then_root_path(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
str_prefix, pathlib_prefix = get_prefixes(fs_kind)
p = TransparentPath("chien")
p2 = p / ".."
assert str(p2) == str_prefix
p2 = TransparentPath()
assert str(p2) == str_prefix
p2 = TransparentPath("/")
if fs_kind == "local":
assert str(p2) == "/"
else:
assert str(p2) == str_prefix
def test_max_size(clean):
"""
testing behavior when reatching maximum size
"""
TransparentPath.caching_max_memory = 0.000070
TransparentPath.caching = "ram"
TransparentPath("tests/data/chat.txt", enable_caching=True,
fs="local").read()
path = TransparentPath("tests/data/groschat.txt",
enable_caching=True,
fs="local")
path.read()
ret = list(TransparentPath.cached_data_dict.keys())
retdata = list(TransparentPath.cached_data_dict.values())
assert len(ret) == 1
assert ret[0] == path.__hash__()
assert retdata[0]["data"] == "grostest"
def test_with_suffix(clean, fs_kind, suffix, expected):
if skip_gcs[fs_kind]:
print("skipped", fs_kind, suffix, expected)
return
init(fs_kind)
p = TransparentPath("chien").with_suffix(suffix)
assert p.suffix == expected
def test_caching_ram(clean, suffix, kwargs):
if reqs_ok:
import pandas as pd
data = pd.DataFrame(columns=["foo", "bar"],
index=["a", "b"],
data=[[1, 2], [3, 4]])
TransparentPath.caching = "ram"
path = TransparentPath(f"tests/data/chien{suffix}",
enable_caching=True,
fs="local")
path.read(**kwargs)
assert all(
TransparentPath.cached_data_dict[path.__hash__()]["data"] == data)
assert all(
TransparentPath(f"tests/data/chien{suffix}",
enable_caching=True,
fs="local").read() == data)
def get_path(fs_kind, suffix):
reload(sys.modules["transparentpath"])
if skip_gcs[fs_kind]:
print("skipped")
return "skipped"
init(fs_kind)
if fs_kind == "local":
local_path = TransparentPath(f"tests/data/chien{suffix}")
pfile = TransparentPath(f"chien{suffix}")
local_path.cp(pfile)
else:
local_path = TransparentPath(f"tests/data/chien{suffix}",
fs_kind="local")
pfile = TransparentPath(f"chien{suffix}")
local_path.put(pfile)
return pfile
def test_mkdir(clean, fs_kind, path):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p = TransparentPath(path)
p.mkdir()
assert p.is_dir()
def test_exists(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p = TransparentPath("chien")
p.touch()
assert p.exist()
assert p.exists()
def test_touch(clean, fs_kind, path):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p = TransparentPath(path)
if p.exists():
p.rm(ignore_kind=True)
p.touch()
assert p.is_file()