def test_mirror_url_source_0(mirror_dirs):
tmpdir, _ = mirror_dirs
mirror = DirectoryMirror(path=tmpdir)
mirror.activate()
with pytest.raises(OfflineError):
load_without_network(force=True)
def test_mirror_url_source_2(mirror_dirs):
tmpdir, _ = mirror_dirs
mirror = DirectoryMirror(path=tmpdir)
mirror.activate(prefetch=True)
# load is enough with prefetch
load(force=True)
load_without_network(force=True)
def test_mirror_url_source_2bis(mirror_dirs):
tmpdir, _ = mirror_dirs
mirror = DirectoryMirror(path=tmpdir)
# load is enough with prefetch
with mirror.prefetch():
load(force=True)
load_without_network(force=True)
def test_mirror_url_source_1(mirror_dirs):
tmpdir, _ = mirror_dirs
mirror = DirectoryMirror(path=tmpdir)
mirror.activate()
# load is not enough, as we are not using the cache ()
load(force=True)
with pytest.raises(OfflineError):
load_without_network(force=True)
def test_mirror_url_source_3(mirror_dirs):
tmpdir, _ = mirror_dirs
mirror = DirectoryMirror(path=tmpdir)
with mirror.prefetch():
load(force=True)
mirror = DirectoryMirror(path=tmpdir)
with mirror:
load_without_network(force=True)
def test_mirror_url_source_multiple_copy(
mirror_dirs,
a1,
b1,
a2,
b2,
):
dir1, dir2 = mirror_dirs
a1 = a1 == "a1"
b1 = b1 == "b1"
a2 = a2 == "a2"
b2 = b2 == "b2"
force = True
mirror_a = DirectoryMirror(path=dir1)
mirror_b = DirectoryMirror(path=dir2)
def contains(mirror, source):
path = mirror.contains(source)
if path:
assert os.path.exists(path)
return path
source = load(force=force)
if a1:
with mirror_a.prefetch():
load(force=force)
if b1:
with mirror_b.prefetch():
load(force=force)
assert contains(mirror_a, source) == a1, "Prefetching failed for a1"
assert contains(mirror_b, source) == b1, "Prefetching failed for b1"
if a2:
mirror_a.prefetch()
if b2:
mirror_b.prefetch()
with mirror_a:
with mirror_b:
assert len(get_active_mirrors()) == 2
source2 = load(force=force)
assert contains(mirror_a, source) == (a1 or a2), "Prefetching failed for a"
assert contains(mirror_b, source) == (b1 or b2), "Prefetching failed for b"
if a1 or b1 or a2 or b2:
assert str(source2) == str(source)
def test_mirror_eod(mirror_dirs):
tmpdir, _ = mirror_dirs
mirror = DirectoryMirror(path=tmpdir)
with mirror.prefetch():
source = load_eod()
purge_cache(matcher=lambda x: not x["path"].endswith(".json"))
with mirror:
source2 = load_eod_without_network()
assert str(source).startswith("EODRetriever("), source
assert str(source2).startswith("EODRetriever("), source2
assert str(source2) == str(source)
def test_mirror_url_source_env_var_2(mirror_dirs):
mirror_dir, _ = mirror_dirs
source = load(force=True)
os.makedirs(f"{mirror_dir}/url/examples", exist_ok=True)
source.save(f"{mirror_dir}/url/examples/test.grib")
origin_prefix = "https://github.com/ecmwf/climetlab/raw/main/docs"
os.environ["CLIMETLAB_MIRROR"] = f"{origin_prefix} file://{mirror_dir}"
m = DirectoryMirror(path=mirror_dir, origin_prefix=origin_prefix)
with m:
source2 = load_without_network(force=True)
assert source.connect_to_mirror(m).resource()
assert str(source) == f"Url({origin_prefix}/examples/test.grib)"
if not IN_GITHUB:
assert str(source2) == str(source)