def test_glob(clean, fs_kind, pattern, expected): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) dic = { "chien": "dir", "chien/chat": "file", "chien/cheval": "dir", "chien/cheval/chouette": "file" } root = TransparentPath() for word in dic: p = root / word p.rm(absent="ignore", ignore_kind=True) for word in dic: p = root / word if dic[word] == "file": p.touch() else: p.mkdir() print(list(TransparentPath("chien").ls())) content = [ str(p).split("chien/")[1] for p in TransparentPath().glob(pattern) ] assert content == expected for word in dic: p = root / word p.rm(absent="ignore", ignore_kind=True)
def test_truediv(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") assert (p1 / "chat") == TransparentPath(cc) assert (p1 / "/chat") == TransparentPath(cc)
def test_gt(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") / "chat" p2 = TransparentPath("chien") assert p1 > p2
def test_append(clean, fs_kind, to_append): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) assert str(TransparentPath("chien").append(to_append)) == str( TransparentPath(f"chien{to_append}"))
def test_equal(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") p2 = TransparentPath("chien") assert p1 == p2
def test_mkdir(clean, fs_kind, path): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p = TransparentPath(path) p.mkdir() assert p.is_dir()
def test_le(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") / "chat" p2 = TransparentPath("chien") p3 = TransparentPath("chien") assert p2 <= p1 assert p3 <= p2
def test_init_gcs_fail( ): # Do not use clean : we do NOT want to execute the rm for fs will have FAILED to setup if skip_gcs["gcs"]: print("skipped") return reinit() with pytest.raises(TPNotADirectoryError): TransparentPath.set_global_fs("gcs", bucket="coucou") reinit()
def test_itruediv(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") p1 /= "chat" assert p1 == TransparentPath(cc) p1 = TransparentPath("chien") p1 /= "/chat" assert p1 == TransparentPath(cc)
def get_path(fs_kind): reload(sys.modules["transparentpath"]) if skip_gcs[fs_kind]: print("skipped") return "skipped" init(fs_kind) pcsv = TransparentPath("chien.json") pcsv.rm(absent="ignore", ignore_kind=True) assert not pcsv.is_file() return pcsv
def test_get(clean): print("test_get") if skip_gcs["gcs"]: print("skipped") return init("gcs") localpath = TransparentPath("chien.txt", fs_kind="local") remotepath = TransparentPath("chien.txt") remotepath.touch() remotepath.get(localpath) assert remotepath.is_file() assert localpath.is_file()
def test_multiproject_1(clean): if skip_gcs["gcs"]: print("skipped") return init("gcs") TransparentPath.set_global_fs("gcs", token=os.environ["GOOGLE_APPLICATION_CREDENTIALS_2"]) p1 = TransparentPath("code_tests_sand") p2 = TransparentPath("code_tests") p3 = TransparentPath("coucou") assert "gcs" in p1.fs_kind assert "gcs" in p2.fs_kind assert p1.fs_kind != p2.fs_kind assert p3.fs_kind == "local"
def test_set_global_fs_then_path_with_gs_failed(clean): if skip_gcs["gcs"]: print("skipped") return init("gcs") with pytest.raises(TPValueError): TransparentPath(f"gs://{bucket + 'chat'}/chien", bucket=bucket) with pytest.raises(TPNotADirectoryError): TransparentPath(f"gs://{bucket + 'chat'}/chien") with pytest.raises(TPValueError): TransparentPath(f"gs://{bucket}/chien", fs="local")
def test_cp(clean, fs_kind1, fs_kind2): print("test_cp", fs_kind1, fs_kind2) if skip_gcs[fs_kind1] or skip_gcs[fs_kind2]: print("skipped") return if fs_kind1 != "local": init(fs_kind1) elif fs_kind2 != "local": init(fs_kind2) path1 = TransparentPath("chien.txt", fs_kind=fs_kind1) path2 = TransparentPath("chien2.txt", fs_kind=fs_kind2) path1.touch() path1.cp(path2) assert path1.is_file() assert path2.is_file()
def test_gcs_path_without_set_global_fs_fail(clean, args, kwargs): if skip_gcs["gcs"]: print("skipped") return with pytest.raises(TPNotADirectoryError): TransparentPath(*args, **kwargs)
def test_contains(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") / "chat" assert "chi" in p1
def test_buckets(clean): if skip_gcs["gcs"]: print("skipped") return init("gcs") assert "code_tests_sand/" in TransparentPath().buckets
def __init__(self, path: Union[str, Path, "TransparentPath"], show=False): self.limits = {} self.path = path if type(path) == str: try: from transparentpath import TransparentPath path = TransparentPath(path) except ImportError: path = Path(path) if not path.is_file(): return if show: logger.info(f"Found threshold file {path}") if hasattr(path, "read"): self.limits = path.read() else: with open(path) as opath: self.limits = load(opath) if show: message = "\n".join( [f"{i}: {self.limits[i]}" for i in self.limits]) logger.info(f"Thresholds are \n{message}") if "coverage" not in self.limits: self.limits["coverage"] = {"min": 0.05} logger.info("Coverage limit was not set. Set to minimum 5%.") elif "min" not in self.limits["coverage"]: self.limits["coverage"]["min"] = 0.05 logger.info("Coverage lower limit was not set. Set to 5%.") elif self.limits["coverage"]["min"] == 0: logger.warning("Coverage limit is set to 0.")
def test_multipleexistenceerror(clean, fs_kind, excep): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) # noinspection PyTypeChecker with pytest.raises(excep): p1 = TransparentPath("chien") p2 = TransparentPath("chien") / "chat" p2.touch() if excep == FileExistsError: p1.touch() else: p1.fs.touch(p1.__fspath__()) TransparentPath("chien").read()
def test_set_global_fs_then_root_path(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) str_prefix, pathlib_prefix = get_prefixes(fs_kind) p = TransparentPath("chien") p2 = p / ".." assert str(p2) == str_prefix p2 = TransparentPath() assert str(p2) == str_prefix p2 = TransparentPath("/") if fs_kind == "local": assert str(p2) == "/" else: assert str(p2) == str_prefix
def test_with_suffix(clean, fs_kind, suffix, expected): if skip_gcs[fs_kind]: print("skipped", fs_kind, suffix, expected) return init(fs_kind) p = TransparentPath("chien").with_suffix(suffix) assert p.suffix == expected
def init_local_class_then_gcs_path(clean): if skip_gcs["gcs"]: print("skipped") return init("local") p = TransparentPath("chien", fs="gcs", bucket=bucket) assert str(p.path) == f"{bucket}/chien" assert str(p) == f"gs://{bucket}/chien" assert p.__fspath__() == f"gs://{bucket}/chien" assert "gcs" in p.fs_kind assert p.fs == TransparentPath.fss["gcs_sandbox-281209"] assert not TransparentPath.unset assert len(TransparentPath.fss) == 2 assert TransparentPath.fs_kind == "local" assert "gcs_sandbox-281209" in list( TransparentPath.fss.keys()) and "local" in list( TransparentPath.fss.keys()) assert isinstance(TransparentPath.fss["gcs_sandbox-281209"], gcsfs.GCSFileSystem) assert isinstance(TransparentPath.fss["local"], LocalFileSystem) reinit() init("local") with pytest.raises(ValueError): TransparentPath(f"gs://{bucket}/chien", bucket=bucket + "chien") with pytest.raises(ValueError): TransparentPath(f"gs://{bucket}/chien", fs="local", bucket=bucket) with pytest.raises(ValueError): TransparentPath(f"gs://{bucket}/chien") p = TransparentPath(f"gs://{bucket}/chien", ) assert str(p.path) == f"{bucket}/chien" assert str(p) == f"gs://{bucket}/chien" assert p.__fspath__() == f"gs://{bucket}/chien" assert "gcs" in p.fs_kind assert p.fs == TransparentPath.fss["gcs_sandbox-281209"] assert not TransparentPath.unset assert len(TransparentPath.fss) == 2 assert TransparentPath.fs_kind == "local" assert "gcs_sandbox-281209" in list( TransparentPath.fss.keys()) and "local" in list( TransparentPath.fss.keys()) assert isinstance(TransparentPath.fss["gcs_sandbox-281209"], gcsfs.GCSFileSystem) assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
def treat_args(self, args_): self.units = args_.units self.activations = args_.activations self.first_layer = args_.first_layer self.shape = args_.shape self.epochs = args_.epochs self.rewrite = args_.rewrite self.convolute = args_.convolute self.input_path = args_.input self.force = args_.force self.verbose = args_.verbose if "tf.keras.datasets" in self.input_path: self.mnist = getattr( tf.keras.datasets, self.input_path.split("tf.keras.datasets.")[1]) else: self.input_path = Path(self.input_path) self.directories = list(self.input_path.glob("/*")) self.rgb = args_.rgb self.loss = args_.loss self.optimizer = getattr(tf.optimizers, args_.optimizer) if args_.optimizer_kwargs == "": self.optimizer_kwargs = {} else: self.optimizer_kwargs = { keyval.split(":")[0]: keyval.split(":")[1] for keyval in args_.optimizer_kwargs.split(",") } for arg in self.optimizer_kwargs: if "%i" in self.optimizer_kwargs[arg]: # noinspection PyUnresolvedReferences self.optimizer_kwargs[arg] = int( self.optimizer_kwargs[arg].replace("%i", "")) if "%f" in self.optimizer_kwargs[arg]: # noinspection PyUnresolvedReferences self.optimizer_kwargs[arg] = float( self.optimizer_kwargs[arg].replace("%f", "")) if "%b" in self.optimizer_kwargs[arg]: # noinspection PyUnresolvedReferences self.optimizer_kwargs[arg] = bool( self.optimizer_kwargs[arg].replace("%b", "")) if isinstance(self.units, str): self.units = [int(u) for u in self.units.split(",")] if isinstance(self.activations, str): self.activations = [ getattr(tf.nn, act) for act in self.activations.split(",") ] if isinstance(self.first_layer, str): self.first_layer = getattr(tf.keras.layers, self.first_layer) if isinstance(self.shape, str): self.shape = tuple([int(u) for u in self.shape.split(",")]) if isinstance(self.epochs, str): self.epochs = int(self.epochs) if isinstance(self.convolute, str): self.convolute = [int(c) for c in self.convolute.split(",")]
def get_path(fs_kind, suffix): reload(sys.modules["transparentpath"]) if skip_gcs[fs_kind]: print("skipped") return "skipped" init(fs_kind) if fs_kind == "local": local_path = TransparentPath(f"tests/data/chien{suffix}") pfile = TransparentPath(f"chien{suffix}") local_path.cp(pfile) else: local_path = TransparentPath(f"tests/data/chien{suffix}", fs_kind="local") pfile = TransparentPath(f"chien{suffix}") local_path.put(pfile) return pfile
def test_touch(clean, fs_kind, path): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p = TransparentPath(path) if p.exists(): p.rm(ignore_kind=True) p.touch() assert p.is_file()
def test_isfile(clean, fs_kind, path1, path2, expected): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath(path1) p1.touch() p2 = TransparentPath(path2) assert p2.is_file() == expected
def test_reload_from_write(clean, path, enable_caching, fs, bucket, mod): """ testing unload and read with args/kwargs """ if reqs_ok: import pandas as pd tp = TransparentPath(path, enable_caching=enable_caching, fs=fs, bucket=bucket) chat = pd.DataFrame(columns=["foo", "bar"], index=["a", "b"], data=[[1, 2], [3, 4]]) TransparentPath.caching = mod path = tp path.write(chat) path.read(index_col=0) if mod == "ram": data_to_test = TransparentPath.cached_data_dict[ path.__hash__()]["data"] else: data_to_test = TransparentPath( TransparentPath.cached_data_dict[path.__hash__()]["file"].name, fs="local").read(index_col=0) assert all(data_to_test == chat) paschat = pd.DataFrame(columns=["bar", "foo"], index=["a", "b"], data=[[5, 7], [6, 2]]) path.write(paschat) if mod == "ram": data_to_test = TransparentPath.cached_data_dict[ path.__hash__()]["data"] else: data_to_test = TransparentPath( TransparentPath.cached_data_dict[path.__hash__()]["file"].name, fs="local").read(index_col=0) assert all(data_to_test == paschat)
def test_path_success(clean, fs_kind, global_init, expected_fs_kind, expected_fs_type, args, kwargs): if skip_gcs[fs_kind]: print("skipped") return if global_init: init(fs_kind) str_prefix, pathlib_prefix = get_prefixes(fs_kind) p = TransparentPath(*args, **kwargs) if "gs://" not in args[0]: assert str(p.path) == f"{pathlib_prefix}/{args[0]}" assert str(p) == f"{str_prefix}/{args[0]}" assert p.__fspath__() == f"{str_prefix}/{args[0]}" else: assert str(p.path) == f"{args[0].replace('gs://', '')}" assert str(p) == f"{args[0]}" assert p.__fspath__() == f"{args[0]}" assert fs_kind in p.fs_kind if global_init: for fs_name in TransparentPath.fss: if expected_fs_kind in fs_name: expected_fs_kind = fs_name break assert p.fs == TransparentPath.fss[expected_fs_kind] assert not TransparentPath.unset if expected_fs_kind == "local": assert len(TransparentPath.fss) == 1 else: assert len(TransparentPath.fss) == 2 assert TransparentPath.fs_kind == fs_kind assert list(TransparentPath.fss.keys())[-1] == expected_fs_kind assert isinstance(TransparentPath.fss[expected_fs_kind], expected_fs_type) else: assert TransparentPath.unset
def init_gcs_class_then_local_path(clean): if skip_gcs["gcs"]: print("skipped") return init("gcs") p = TransparentPath("chien", fs="local") assert str(p.path) == f"{os.getcwd()}/chien" assert str(p) == f"{os.getcwd()}/chien" assert p.__fspath__() == f"{os.getcwd()}/chien" assert p.fs_kind == "local" assert p.fs == TransparentPath.fss["local"] assert not TransparentPath.unset assert len(TransparentPath.fss) == 2 assert "gcs" in TransparentPath.fs_kind assert "gcs_sandbox-281209" in list( TransparentPath.fss.keys()) and "local" in list( TransparentPath.fss.keys()) assert isinstance(TransparentPath.fss["gcs_sandbox-281209"], gcsfs.GCSFileSystem) assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
def test_cd(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) root = TransparentPath() (root / "chien").mkdir() (root / "chien" / "chat").touch() (root / "chien" / "cheval").mkdir() (root / "chien" / "cheval" / "chouette").touch() root.cd("chien") assert root == TransparentPath("chien") root.cd("..") assert root == TransparentPath()