def mirror_dirs():
"""Setup for mirror tests:
- a temporary cache dir,
- temporary settings,
- two temporary directories
"""
with temp_directory() as cachedir:
with settings.temporary():
settings.set("cache-directory", cachedir)
with temp_directory() as mirrordir:
with temp_directory() as mirrordir2:
_reset_mirrors(use_env_var=False)
yield mirrordir, mirrordir2
def test_url_source_check_out_of_date():
def load():
load_source(
"url",
"http://download.ecmwf.int/test-data/metview/gallery/temp.bufr",
)
with temp_directory() as tmpdir:
with settings.temporary():
settings.set("cache-directory", tmpdir)
load()
settings.set("check-out-of-date-urls", False)
with network_off():
load()
def test_temporary():
settings.reset()
settings.set("styles-directories", "/c", "/d")
settings.set("plotting-options", {"width": 400})
with settings.temporary("plotting-options", {"width": 100}):
assert settings.get("styles-directories") == ["/c", "/d"]
assert settings.get("plotting-options") == {"width": 100}, settings.get(
"plotting-options"
)
settings.set("plotting-options", {"width": 200})
assert settings.get("plotting-options") == {"width": 200}
settings.reset()
assert settings.get("plotting-options") == {}
settings.set("plotting-options", {"width": 400})
settings.set("styles-directories", "/c", "/d")
settings.reset()
assert settings.get("plotting-options") == {}
def do_settings(self, args):
"""
Display or change CliMetLab settings.
See https://climetlab.readthedocs.io/guide/settings.html for more details.
Examples: climetlab settings cache-directory /big-disk/climetlab-cache
"""
from climetlab import settings
words = args.args
if len(words) == 0:
if args.json:
result = {}
for f in settings.dump():
result[f[0]] = f[1]
print(json.dumps(result, indent=4, sort_keys=True))
return
print_table((f[0], f[1]) for f in settings.dump())
return
if len(words) == 1:
name = words[0]
print(settings.get(name))
return
if len(words) == 2:
name = words[0]
value = words[1]
settings.set(name, value)
if len(words) == 3:
name = words[0]
key = words[1]
value = words[2]
settings.set(name, {key: value})
def test_cache_2():
with temp_directory() as tmpdir:
with settings.temporary():
settings.set("cache-directory", tmpdir)
settings.set("maximum-cache-size", "5MB")
settings.set("number-of-download-threads", 5)
assert cache_size() == 0
load_source(
"url-pattern",
"https://get.ecmwf.int/test-data/climetlab/1mb-{n}.bin",
{
"n": [0, 1, 2, 3, 4],
},
)
assert cache_size() == 5 * 1024 * 1024, cache_size()
cnt = 0
for i, f in enumerate(cache_entries()):
print("FILE", i, f)
cnt += 1
assert cnt == 5, f"Files in cache database: {cnt}"
load_source(
"url-pattern",
"https://get.ecmwf.int/test-data/climetlab/1mb-{n}.bin",
{
"n": [5, 6, 7, 8, 9],
},
)
assert cache_size(
) == 5 * 1024 * 1024, cache_size() / 1024.0 / 1024.0
cnt = 0
for i, f in enumerate(cache_entries()):
print("FILE", i, f)
cnt += 1
assert cnt == 5, f"Files in cache database: {cnt}"
cnt = 0
for n in os.listdir(tmpdir):
if n.startswith("cache-") and n.endswith(".db"):
continue
cnt += 1
assert cnt == 5, f"Files in cache directory: {cnt}"
def test_numbers():
with temp_directory() as tmpdir:
with settings.temporary("cache-directory", tmpdir):
settings.set("url-download-timeout", 30)
assert settings.get("url-download-timeout") == 30
settings.set("url-download-timeout", "30")
assert settings.get("url-download-timeout") == 30
settings.set("url-download-timeout", "30s")
assert settings.get("url-download-timeout") == 30
settings.set("url-download-timeout", "2m")
assert settings.get("url-download-timeout") == 120
settings.set("url-download-timeout", "10h")
assert settings.get("url-download-timeout") == 36000
settings.set("url-download-timeout", "7d")
assert settings.get("url-download-timeout") == 7 * 24 * 3600
with pytest.raises(ValueError):
settings.set("url-download-timeout", "1x")
settings.set("maximum-cache-size", "1")
assert settings.get("maximum-cache-size") == 1
settings.set("maximum-cache-size", "1k")
assert settings.get("maximum-cache-size") == 1024
settings.set("maximum-cache-size", "1kb")
assert settings.get("maximum-cache-size") == 1024
settings.set("maximum-cache-size", "1k")
assert settings.get("maximum-cache-size") == 1024
settings.set("maximum-cache-size", "1kb")
assert settings.get("maximum-cache-size") == 1024
settings.set("maximum-cache-size", "1K")
assert settings.get("maximum-cache-size") == 1024
settings.set("maximum-cache-size", "1M")
assert settings.get("maximum-cache-size") == 1024 * 1024
settings.set("maximum-cache-size", "1G")
assert settings.get("maximum-cache-size") == 1024 * 1024 * 1024
settings.set("maximum-cache-size", "1T")
assert settings.get("maximum-cache-size") == 1024 * 1024 * 1024 * 1024
settings.set("maximum-cache-size", "1P")
assert (
settings.get("maximum-cache-size") == 1024 * 1024 * 1024 * 1024 * 1024
)
settings.set("maximum-cache-size", None)
assert settings.get("maximum-cache-size") is None
settings.set("maximum-cache-disk-usage", "2%")
assert settings.get("maximum-cache-disk-usage") == 2
def test_settings():
with settings.temporary():
settings.reset()
assert settings.get("plotting-options") == {}, "Check 1"
settings.set("plotting-options", width=400)
assert settings.get("plotting-options") == {"width": 400}
settings.reset("plotting-options")
assert settings.get("plotting-options") == {}, "Check 2"
settings.set("plotting-options", {"width": 400})
assert settings.get("plotting-options") == {"width": 400}
settings.reset()
assert settings.get("plotting-options") == {}, "Check 3"
with pytest.raises(TypeError):
settings.set("plotting-options", 3)
settings.set("styles-directories", ["/a", "/b"])
assert settings.get("styles-directories") == ["/a", "/b"]
settings.set("styles-directories", "/c", "/d")
assert settings.get("styles-directories") == ["/c", "/d"]
with pytest.raises(KeyError):
settings.set("test", 42)
with pytest.raises(KeyError):
settings.get("test")
with pytest.raises(ValueError):
settings.set("url-download-timeout", "1M")
def test_cache_4():
with settings.temporary():
settings.set("cache-directory", "/Volumes/RAMDisk/climetlab")
settings.set("maximum-cache-disk-usage", "90%")
for n in range(10):
load_source("dummy-source", "zeros", size=100 * 1024 * 1024, n=n)
def test_cache_2():
with temp_directory() as tmpdir:
with settings.temporary():
settings.set("cache-directory", tmpdir)
settings.set("maximum-cache-size", "5MB")
settings.set("number-of-download-threads", 5)
assert cache_size() == 0
load_source(
"url-pattern",
f"{TEST_DATA_URL}/input/" + "1mb-{n}.bin",
{
"n": [0, 1, 2, 3, 4],
},
)
cachesize = cache_size()
expected = 5 * 1024 * 1024
if cachesize != expected:
print(json.dumps(dump_cache_database(), indent=4))
assert cachesize == expected, ("before", cachesize / 1024.0 / 1024.0)
cnt = 0
for i, f in enumerate(cache_entries()):
# print("FILE", i, f)
cnt += 1
if cnt != 5:
print(json.dumps(dump_cache_database(), indent=4))
assert cnt == 5, f"Files in cache database (before): {cnt}"
load_source(
"url-pattern",
f"{TEST_DATA_URL}/input/" + "1mb-{n}.bin",
{
"n": [5, 6, 7, 8, 9],
},
)
cachesize = cache_size()
expected = 5 * 1024 * 1024
if cachesize != expected:
print(json.dumps(dump_cache_database(), indent=4))
assert cachesize == expected, ("after", cachesize / 1024.0 / 1024.0)
cnt = 0
for i, f in enumerate(cache_entries()):
LOG.debug("FILE %s %s", i, f)
cnt += 1
if cnt != 5:
print(json.dumps(dump_cache_database(), indent=4))
assert cnt == 5, f"Files in cache database (after): {cnt}"
cnt = 0
for n in os.listdir(tmpdir):
if n.startswith("cache-") and n.endswith(".db"):
continue
cnt += 1
if cnt != 5:
print(json.dumps(dump_cache_database(), indent=4))
assert cnt == 5, f"Files in cache directory: {cnt}"
