def test_glob(clean, fs_kind, pattern, expected):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
dic = {
"chien": "dir",
"chien/chat": "file",
"chien/cheval": "dir",
"chien/cheval/chouette": "file"
}
root = TransparentPath()
for word in dic:
p = root / word
p.rm(absent="ignore", ignore_kind=True)
for word in dic:
p = root / word
if dic[word] == "file":
p.touch()
else:
p.mkdir()
print(list(TransparentPath("chien").ls()))
content = [
str(p).split("chien/")[1] for p in TransparentPath().glob(pattern)
]
assert content == expected
for word in dic:
p = root / word
p.rm(absent="ignore", ignore_kind=True)
def test_truediv(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien")
assert (p1 / "chat") == TransparentPath(cc)
assert (p1 / "/chat") == TransparentPath(cc)
def test_gt(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien") / "chat"
p2 = TransparentPath("chien")
assert p1 > p2
def test_append(clean, fs_kind, to_append):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
assert str(TransparentPath("chien").append(to_append)) == str(
TransparentPath(f"chien{to_append}"))
def test_equal(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien")
p2 = TransparentPath("chien")
assert p1 == p2
def test_mkdir(clean, fs_kind, path):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p = TransparentPath(path)
p.mkdir()
assert p.is_dir()
def test_le(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien") / "chat"
p2 = TransparentPath("chien")
p3 = TransparentPath("chien")
assert p2 <= p1
assert p3 <= p2
def test_init_gcs_fail(
): # Do not use clean : we do NOT want to execute the rm for fs will have FAILED to setup
if skip_gcs["gcs"]:
print("skipped")
return
reinit()
with pytest.raises(TPNotADirectoryError):
TransparentPath.set_global_fs("gcs", bucket="coucou")
reinit()
def test_itruediv(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien")
p1 /= "chat"
assert p1 == TransparentPath(cc)
p1 = TransparentPath("chien")
p1 /= "/chat"
assert p1 == TransparentPath(cc)
def get_path(fs_kind):
reload(sys.modules["transparentpath"])
if skip_gcs[fs_kind]:
print("skipped")
return "skipped"
init(fs_kind)
pcsv = TransparentPath("chien.json")
pcsv.rm(absent="ignore", ignore_kind=True)
assert not pcsv.is_file()
return pcsv
def test_get(clean):
print("test_get")
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
localpath = TransparentPath("chien.txt", fs_kind="local")
remotepath = TransparentPath("chien.txt")
remotepath.touch()
remotepath.get(localpath)
assert remotepath.is_file()
assert localpath.is_file()
def test_multiproject_1(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
TransparentPath.set_global_fs("gcs", token=os.environ["GOOGLE_APPLICATION_CREDENTIALS_2"])
p1 = TransparentPath("code_tests_sand")
p2 = TransparentPath("code_tests")
p3 = TransparentPath("coucou")
assert "gcs" in p1.fs_kind
assert "gcs" in p2.fs_kind
assert p1.fs_kind != p2.fs_kind
assert p3.fs_kind == "local"
def test_set_global_fs_then_path_with_gs_failed(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
with pytest.raises(TPValueError):
TransparentPath(f"gs://{bucket + 'chat'}/chien", bucket=bucket)
with pytest.raises(TPNotADirectoryError):
TransparentPath(f"gs://{bucket + 'chat'}/chien")
with pytest.raises(TPValueError):
TransparentPath(f"gs://{bucket}/chien", fs="local")
def test_cp(clean, fs_kind1, fs_kind2):
print("test_cp", fs_kind1, fs_kind2)
if skip_gcs[fs_kind1] or skip_gcs[fs_kind2]:
print("skipped")
return
if fs_kind1 != "local":
init(fs_kind1)
elif fs_kind2 != "local":
init(fs_kind2)
path1 = TransparentPath("chien.txt", fs_kind=fs_kind1)
path2 = TransparentPath("chien2.txt", fs_kind=fs_kind2)
path1.touch()
path1.cp(path2)
assert path1.is_file()
assert path2.is_file()
def test_gcs_path_without_set_global_fs_fail(clean, args, kwargs):
if skip_gcs["gcs"]:
print("skipped")
return
with pytest.raises(TPNotADirectoryError):
TransparentPath(*args, **kwargs)
def test_contains(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath("chien") / "chat"
assert "chi" in p1
def test_buckets(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
assert "code_tests_sand/" in TransparentPath().buckets
def __init__(self, path: Union[str, Path, "TransparentPath"], show=False):
self.limits = {}
self.path = path
if type(path) == str:
try:
from transparentpath import TransparentPath
path = TransparentPath(path)
except ImportError:
path = Path(path)
if not path.is_file():
return
if show:
logger.info(f"Found threshold file {path}")
if hasattr(path, "read"):
self.limits = path.read()
else:
with open(path) as opath:
self.limits = load(opath)
if show:
message = "\n".join(
[f"{i}: {self.limits[i]}" for i in self.limits])
logger.info(f"Thresholds are \n{message}")
if "coverage" not in self.limits:
self.limits["coverage"] = {"min": 0.05}
logger.info("Coverage limit was not set. Set to minimum 5%.")
elif "min" not in self.limits["coverage"]:
self.limits["coverage"]["min"] = 0.05
logger.info("Coverage lower limit was not set. Set to 5%.")
elif self.limits["coverage"]["min"] == 0:
logger.warning("Coverage limit is set to 0.")
def test_multipleexistenceerror(clean, fs_kind, excep):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
# noinspection PyTypeChecker
with pytest.raises(excep):
p1 = TransparentPath("chien")
p2 = TransparentPath("chien") / "chat"
p2.touch()
if excep == FileExistsError:
p1.touch()
else:
p1.fs.touch(p1.__fspath__())
TransparentPath("chien").read()
def test_set_global_fs_then_root_path(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
str_prefix, pathlib_prefix = get_prefixes(fs_kind)
p = TransparentPath("chien")
p2 = p / ".."
assert str(p2) == str_prefix
p2 = TransparentPath()
assert str(p2) == str_prefix
p2 = TransparentPath("/")
if fs_kind == "local":
assert str(p2) == "/"
else:
assert str(p2) == str_prefix
def test_with_suffix(clean, fs_kind, suffix, expected):
if skip_gcs[fs_kind]:
print("skipped", fs_kind, suffix, expected)
return
init(fs_kind)
p = TransparentPath("chien").with_suffix(suffix)
assert p.suffix == expected
def init_local_class_then_gcs_path(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("local")
p = TransparentPath("chien", fs="gcs", bucket=bucket)
assert str(p.path) == f"{bucket}/chien"
assert str(p) == f"gs://{bucket}/chien"
assert p.__fspath__() == f"gs://{bucket}/chien"
assert "gcs" in p.fs_kind
assert p.fs == TransparentPath.fss["gcs_sandbox-281209"]
assert not TransparentPath.unset
assert len(TransparentPath.fss) == 2
assert TransparentPath.fs_kind == "local"
assert "gcs_sandbox-281209" in list(
TransparentPath.fss.keys()) and "local" in list(
TransparentPath.fss.keys())
assert isinstance(TransparentPath.fss["gcs_sandbox-281209"],
gcsfs.GCSFileSystem)
assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
reinit()
init("local")
with pytest.raises(ValueError):
TransparentPath(f"gs://{bucket}/chien", bucket=bucket + "chien")
with pytest.raises(ValueError):
TransparentPath(f"gs://{bucket}/chien", fs="local", bucket=bucket)
with pytest.raises(ValueError):
TransparentPath(f"gs://{bucket}/chien")
p = TransparentPath(f"gs://{bucket}/chien", )
assert str(p.path) == f"{bucket}/chien"
assert str(p) == f"gs://{bucket}/chien"
assert p.__fspath__() == f"gs://{bucket}/chien"
assert "gcs" in p.fs_kind
assert p.fs == TransparentPath.fss["gcs_sandbox-281209"]
assert not TransparentPath.unset
assert len(TransparentPath.fss) == 2
assert TransparentPath.fs_kind == "local"
assert "gcs_sandbox-281209" in list(
TransparentPath.fss.keys()) and "local" in list(
TransparentPath.fss.keys())
assert isinstance(TransparentPath.fss["gcs_sandbox-281209"],
gcsfs.GCSFileSystem)
assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
def treat_args(self, args_):
self.units = args_.units
self.activations = args_.activations
self.first_layer = args_.first_layer
self.shape = args_.shape
self.epochs = args_.epochs
self.rewrite = args_.rewrite
self.convolute = args_.convolute
self.input_path = args_.input
self.force = args_.force
self.verbose = args_.verbose
if "tf.keras.datasets" in self.input_path:
self.mnist = getattr(
tf.keras.datasets,
self.input_path.split("tf.keras.datasets.")[1])
else:
self.input_path = Path(self.input_path)
self.directories = list(self.input_path.glob("/*"))
self.rgb = args_.rgb
self.loss = args_.loss
self.optimizer = getattr(tf.optimizers, args_.optimizer)
if args_.optimizer_kwargs == "":
self.optimizer_kwargs = {}
else:
self.optimizer_kwargs = {
keyval.split(":")[0]: keyval.split(":")[1]
for keyval in args_.optimizer_kwargs.split(",")
}
for arg in self.optimizer_kwargs:
if "%i" in self.optimizer_kwargs[arg]:
# noinspection PyUnresolvedReferences
self.optimizer_kwargs[arg] = int(
self.optimizer_kwargs[arg].replace("%i", ""))
if "%f" in self.optimizer_kwargs[arg]:
# noinspection PyUnresolvedReferences
self.optimizer_kwargs[arg] = float(
self.optimizer_kwargs[arg].replace("%f", ""))
if "%b" in self.optimizer_kwargs[arg]:
# noinspection PyUnresolvedReferences
self.optimizer_kwargs[arg] = bool(
self.optimizer_kwargs[arg].replace("%b", ""))
if isinstance(self.units, str):
self.units = [int(u) for u in self.units.split(",")]
if isinstance(self.activations, str):
self.activations = [
getattr(tf.nn, act) for act in self.activations.split(",")
]
if isinstance(self.first_layer, str):
self.first_layer = getattr(tf.keras.layers, self.first_layer)
if isinstance(self.shape, str):
self.shape = tuple([int(u) for u in self.shape.split(",")])
if isinstance(self.epochs, str):
self.epochs = int(self.epochs)
if isinstance(self.convolute, str):
self.convolute = [int(c) for c in self.convolute.split(",")]
def get_path(fs_kind, suffix):
reload(sys.modules["transparentpath"])
if skip_gcs[fs_kind]:
print("skipped")
return "skipped"
init(fs_kind)
if fs_kind == "local":
local_path = TransparentPath(f"tests/data/chien{suffix}")
pfile = TransparentPath(f"chien{suffix}")
local_path.cp(pfile)
else:
local_path = TransparentPath(f"tests/data/chien{suffix}",
fs_kind="local")
pfile = TransparentPath(f"chien{suffix}")
local_path.put(pfile)
return pfile
def test_touch(clean, fs_kind, path):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p = TransparentPath(path)
if p.exists():
p.rm(ignore_kind=True)
p.touch()
assert p.is_file()
def test_isfile(clean, fs_kind, path1, path2, expected):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
p1 = TransparentPath(path1)
p1.touch()
p2 = TransparentPath(path2)
assert p2.is_file() == expected
def test_reload_from_write(clean, path, enable_caching, fs, bucket, mod):
"""
testing unload and read with args/kwargs
"""
if reqs_ok:
import pandas as pd
tp = TransparentPath(path,
enable_caching=enable_caching,
fs=fs,
bucket=bucket)
chat = pd.DataFrame(columns=["foo", "bar"],
index=["a", "b"],
data=[[1, 2], [3, 4]])
TransparentPath.caching = mod
path = tp
path.write(chat)
path.read(index_col=0)
if mod == "ram":
data_to_test = TransparentPath.cached_data_dict[
path.__hash__()]["data"]
else:
data_to_test = TransparentPath(
TransparentPath.cached_data_dict[path.__hash__()]["file"].name,
fs="local").read(index_col=0)
assert all(data_to_test == chat)
paschat = pd.DataFrame(columns=["bar", "foo"],
index=["a", "b"],
data=[[5, 7], [6, 2]])
path.write(paschat)
if mod == "ram":
data_to_test = TransparentPath.cached_data_dict[
path.__hash__()]["data"]
else:
data_to_test = TransparentPath(
TransparentPath.cached_data_dict[path.__hash__()]["file"].name,
fs="local").read(index_col=0)
assert all(data_to_test == paschat)
def test_path_success(clean, fs_kind, global_init, expected_fs_kind,
expected_fs_type, args, kwargs):
if skip_gcs[fs_kind]:
print("skipped")
return
if global_init:
init(fs_kind)
str_prefix, pathlib_prefix = get_prefixes(fs_kind)
p = TransparentPath(*args, **kwargs)
if "gs://" not in args[0]:
assert str(p.path) == f"{pathlib_prefix}/{args[0]}"
assert str(p) == f"{str_prefix}/{args[0]}"
assert p.__fspath__() == f"{str_prefix}/{args[0]}"
else:
assert str(p.path) == f"{args[0].replace('gs://', '')}"
assert str(p) == f"{args[0]}"
assert p.__fspath__() == f"{args[0]}"
assert fs_kind in p.fs_kind
if global_init:
for fs_name in TransparentPath.fss:
if expected_fs_kind in fs_name:
expected_fs_kind = fs_name
break
assert p.fs == TransparentPath.fss[expected_fs_kind]
assert not TransparentPath.unset
if expected_fs_kind == "local":
assert len(TransparentPath.fss) == 1
else:
assert len(TransparentPath.fss) == 2
assert TransparentPath.fs_kind == fs_kind
assert list(TransparentPath.fss.keys())[-1] == expected_fs_kind
assert isinstance(TransparentPath.fss[expected_fs_kind],
expected_fs_type)
else:
assert TransparentPath.unset
def init_gcs_class_then_local_path(clean):
if skip_gcs["gcs"]:
print("skipped")
return
init("gcs")
p = TransparentPath("chien", fs="local")
assert str(p.path) == f"{os.getcwd()}/chien"
assert str(p) == f"{os.getcwd()}/chien"
assert p.__fspath__() == f"{os.getcwd()}/chien"
assert p.fs_kind == "local"
assert p.fs == TransparentPath.fss["local"]
assert not TransparentPath.unset
assert len(TransparentPath.fss) == 2
assert "gcs" in TransparentPath.fs_kind
assert "gcs_sandbox-281209" in list(
TransparentPath.fss.keys()) and "local" in list(
TransparentPath.fss.keys())
assert isinstance(TransparentPath.fss["gcs_sandbox-281209"],
gcsfs.GCSFileSystem)
assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
def test_cd(clean, fs_kind):
if skip_gcs[fs_kind]:
print("skipped")
return
init(fs_kind)
root = TransparentPath()
(root / "chien").mkdir()
(root / "chien" / "chat").touch()
(root / "chien" / "cheval").mkdir()
(root / "chien" / "cheval" / "chouette").touch()
root.cd("chien")
assert root == TransparentPath("chien")
root.cd("..")
assert root == TransparentPath()
