def clean(pytestconfig): token_file = Path("cred.json") token_file2 = Path("cred2.json") token = pytestconfig.getoption("token") token2 = pytestconfig.getoption("token2") process_token(token, token_file) process_token(token2, token_file2) TransparentPath._do_update_cache = True TransparentPath._do_check = True before_init() yield TransparentPath._do_update_cache = False TransparentPath._do_check = False path1 = TransparentPath("chien") path2 = TransparentPath("chien2") suffixes = [ "", ".zip", ".txt", ".json", ".csv", ".parquet", ".hdf5", ".xlsx" ] for suffix in suffixes: path1.with_suffix(suffix).rm(recursive=True, ignore_kind=True, absent="ignore") path1.with_suffix(suffix).rm(recursive=True, ignore_kind=True, absent="ignore") path2.with_suffix(suffix).rm(recursive=True, ignore_kind=True, absent="ignore") path2.with_suffix(suffix).rm(recursive=True, ignore_kind=True, absent="ignore") TransparentPath._do_update_cache = True TransparentPath._do_check = True reinit()
def test_glob(clean, fs_kind, pattern, expected): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) dic = { "chien": "dir", "chien/chat": "file", "chien/cheval": "dir", "chien/cheval/chouette": "file" } root = TransparentPath() for word in dic: p = root / word p.rm(absent="ignore", ignore_kind=True) for word in dic: p = root / word if dic[word] == "file": p.touch() else: p.mkdir() print(list(TransparentPath("chien").ls())) content = [ str(p).split("chien/")[1] for p in TransparentPath().glob(pattern) ] assert content == expected for word in dic: p = root / word p.rm(absent="ignore", ignore_kind=True)
def test_append(clean, fs_kind, to_append): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) assert str(TransparentPath("chien").append(to_append)) == str( TransparentPath(f"chien{to_append}"))
def test_equal(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") p2 = TransparentPath("chien") assert p1 == p2
def test_gt(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") / "chat" p2 = TransparentPath("chien") assert p1 > p2
def test_truediv(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") assert (p1 / "chat") == TransparentPath(cc) assert (p1 / "/chat") == TransparentPath(cc)
def test_isfile(clean, fs_kind, path1, path2, expected): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath(path1) p1.touch() p2 = TransparentPath(path2) assert p2.is_file() == expected
def test_le(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") / "chat" p2 = TransparentPath("chien") p3 = TransparentPath("chien") assert p2 <= p1 assert p3 <= p2
def test_itruediv(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") p1 /= "chat" assert p1 == TransparentPath(cc) p1 = TransparentPath("chien") p1 /= "/chat" assert p1 == TransparentPath(cc)
def test_multiproject_1(clean): if skip_gcs["gcs"]: print("skipped") return init("gcs") TransparentPath.set_global_fs("gcs", token=os.environ["GOOGLE_APPLICATION_CREDENTIALS_2"]) p1 = TransparentPath("code_tests_sand") p2 = TransparentPath("code_tests") p3 = TransparentPath("coucou") assert "gcs" in p1.fs_kind assert "gcs" in p2.fs_kind assert p1.fs_kind != p2.fs_kind assert p3.fs_kind == "local"
def test_put(clean): print("test_put") if skip_gcs["gcs"]: print("skipped") return init("gcs") TransparentPath.show_state() localpath = TransparentPath("chien.txt", fs_kind="local") remotepath = TransparentPath("chien.txt") localpath.touch() localpath.put(remotepath) assert localpath.is_file() assert remotepath.is_file()
def test_set_global_fs_then_path_with_gs_failed(clean): if skip_gcs["gcs"]: print("skipped") return init("gcs") with pytest.raises(TPValueError): TransparentPath(f"gs://{bucket + 'chat'}/chien", bucket=bucket) with pytest.raises(TPNotADirectoryError): TransparentPath(f"gs://{bucket + 'chat'}/chien") with pytest.raises(TPValueError): TransparentPath(f"gs://{bucket}/chien", fs="local")
def init_local_class_then_gcs_path(clean): if skip_gcs["gcs"]: print("skipped") return init("local") p = TransparentPath("chien", fs="gcs", bucket=bucket) assert str(p.path) == f"{bucket}/chien" assert str(p) == f"gs://{bucket}/chien" assert p.__fspath__() == f"gs://{bucket}/chien" assert "gcs" in p.fs_kind assert p.fs == TransparentPath.fss["gcs_sandbox-281209"] assert not TransparentPath.unset assert len(TransparentPath.fss) == 2 assert TransparentPath.fs_kind == "local" assert "gcs_sandbox-281209" in list( TransparentPath.fss.keys()) and "local" in list( TransparentPath.fss.keys()) assert isinstance(TransparentPath.fss["gcs_sandbox-281209"], gcsfs.GCSFileSystem) assert isinstance(TransparentPath.fss["local"], LocalFileSystem) reinit() init("local") with pytest.raises(ValueError): TransparentPath(f"gs://{bucket}/chien", bucket=bucket + "chien") with pytest.raises(ValueError): TransparentPath(f"gs://{bucket}/chien", fs="local", bucket=bucket) with pytest.raises(ValueError): TransparentPath(f"gs://{bucket}/chien") p = TransparentPath(f"gs://{bucket}/chien", ) assert str(p.path) == f"{bucket}/chien" assert str(p) == f"gs://{bucket}/chien" assert p.__fspath__() == f"gs://{bucket}/chien" assert "gcs" in p.fs_kind assert p.fs == TransparentPath.fss["gcs_sandbox-281209"] assert not TransparentPath.unset assert len(TransparentPath.fss) == 2 assert TransparentPath.fs_kind == "local" assert "gcs_sandbox-281209" in list( TransparentPath.fss.keys()) and "local" in list( TransparentPath.fss.keys()) assert isinstance(TransparentPath.fss["gcs_sandbox-281209"], gcsfs.GCSFileSystem) assert isinstance(TransparentPath.fss["local"], LocalFileSystem)
def test_rm(clean, fs_kind, path1, path2, kwargs, expected): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath(path1) p2 = TransparentPath(path2) if path1 != "": p1.touch() if expected is not None: with pytest.raises(expected): p2.rm(**kwargs) else: p2.rm(**kwargs) assert not p2.exists()
def test_cd(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) root = TransparentPath() (root / "chien").mkdir() (root / "chien" / "chat").touch() (root / "chien" / "cheval").mkdir() (root / "chien" / "cheval" / "chouette").touch() root.cd("chien") assert root == TransparentPath("chien") root.cd("..") assert root == TransparentPath()
def test_gcs_path_without_set_global_fs_fail(clean, args, kwargs): if skip_gcs["gcs"]: print("skipped") return with pytest.raises(TPNotADirectoryError): TransparentPath(*args, **kwargs)
def test_contains(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p1 = TransparentPath("chien") / "chat" assert "chi" in p1
def __init__(self, path: Union[str, Path, "TransparentPath"], show=False): self.limits = {} self.path = path if type(path) == str: try: from transparentpath import TransparentPath path = TransparentPath(path) except ImportError: path = Path(path) if not path.is_file(): return if show: logger.info(f"Found threshold file {path}") if hasattr(path, "read"): self.limits = path.read() else: with open(path) as opath: self.limits = load(opath) if show: message = "\n".join( [f"{i}: {self.limits[i]}" for i in self.limits]) logger.info(f"Thresholds are \n{message}") if "coverage" not in self.limits: self.limits["coverage"] = {"min": 0.05} logger.info("Coverage limit was not set. Set to minimum 5%.") elif "min" not in self.limits["coverage"]: self.limits["coverage"]["min"] = 0.05 logger.info("Coverage lower limit was not set. Set to 5%.") elif self.limits["coverage"]["min"] == 0: logger.warning("Coverage limit is set to 0.")
def test_cp(clean, fs_kind1, fs_kind2): print("test_cp", fs_kind1, fs_kind2) if skip_gcs[fs_kind1] or skip_gcs[fs_kind2]: print("skipped") return if fs_kind1 != "local": init(fs_kind1) elif fs_kind2 != "local": init(fs_kind2) path1 = TransparentPath("chien.txt", fs_kind=fs_kind1) path2 = TransparentPath("chien2.txt", fs_kind=fs_kind2) path1.touch() path1.cp(path2) assert path1.is_file() assert path2.is_file()
def test_multipleexistenceerror(clean, fs_kind, excep): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) # noinspection PyTypeChecker with pytest.raises(excep): p1 = TransparentPath("chien") p2 = TransparentPath("chien") / "chat" p2.touch() if excep == FileExistsError: p1.touch() else: p1.fs.touch(p1.__fspath__()) TransparentPath("chien").read()
def test_buckets(clean): if skip_gcs["gcs"]: print("skipped") return init("gcs") assert "code_tests_sand/" in TransparentPath().buckets
def test_zipfile(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) if fs_kind == "local": data_path = TransparentPath("tests/data/chien.zip") else: local_path = TransparentPath("tests/data/chien.zip", fs_kind="local") data_path = TransparentPath("chien.zip") local_path.put(data_path) zf = zipfile.ZipFile(data_path) text1 = zf.open("text.txt") text2 = zf.open("text 2.txt") assert text1.read() == b'chien\n' assert text2.read() == b'chat\n'
def test_set_global_fs_then_root_path(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) str_prefix, pathlib_prefix = get_prefixes(fs_kind) p = TransparentPath("chien") p2 = p / ".." assert str(p2) == str_prefix p2 = TransparentPath() assert str(p2) == str_prefix p2 = TransparentPath("/") if fs_kind == "local": assert str(p2) == "/" else: assert str(p2) == str_prefix
def test_max_size(clean): """ testing behavior when reatching maximum size """ TransparentPath.caching_max_memory = 0.000070 TransparentPath.caching = "ram" TransparentPath("tests/data/chat.txt", enable_caching=True, fs="local").read() path = TransparentPath("tests/data/groschat.txt", enable_caching=True, fs="local") path.read() ret = list(TransparentPath.cached_data_dict.keys()) retdata = list(TransparentPath.cached_data_dict.values()) assert len(ret) == 1 assert ret[0] == path.__hash__() assert retdata[0]["data"] == "grostest"
def test_with_suffix(clean, fs_kind, suffix, expected): if skip_gcs[fs_kind]: print("skipped", fs_kind, suffix, expected) return init(fs_kind) p = TransparentPath("chien").with_suffix(suffix) assert p.suffix == expected
def test_caching_ram(clean, suffix, kwargs): if reqs_ok: import pandas as pd data = pd.DataFrame(columns=["foo", "bar"], index=["a", "b"], data=[[1, 2], [3, 4]]) TransparentPath.caching = "ram" path = TransparentPath(f"tests/data/chien{suffix}", enable_caching=True, fs="local") path.read(**kwargs) assert all( TransparentPath.cached_data_dict[path.__hash__()]["data"] == data) assert all( TransparentPath(f"tests/data/chien{suffix}", enable_caching=True, fs="local").read() == data)
def get_path(fs_kind, suffix): reload(sys.modules["transparentpath"]) if skip_gcs[fs_kind]: print("skipped") return "skipped" init(fs_kind) if fs_kind == "local": local_path = TransparentPath(f"tests/data/chien{suffix}") pfile = TransparentPath(f"chien{suffix}") local_path.cp(pfile) else: local_path = TransparentPath(f"tests/data/chien{suffix}", fs_kind="local") pfile = TransparentPath(f"chien{suffix}") local_path.put(pfile) return pfile
def test_mkdir(clean, fs_kind, path): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p = TransparentPath(path) p.mkdir() assert p.is_dir()
def test_exists(clean, fs_kind): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p = TransparentPath("chien") p.touch() assert p.exist() assert p.exists()
def test_touch(clean, fs_kind, path): if skip_gcs[fs_kind]: print("skipped") return init(fs_kind) p = TransparentPath(path) if p.exists(): p.rm(ignore_kind=True) p.touch() assert p.is_file()