def test_settings(): with settings.temporary(): settings.reset() assert settings.get("plotting-options") == {}, "Check 1" settings.set("plotting-options", width=400) assert settings.get("plotting-options") == {"width": 400} settings.reset("plotting-options") assert settings.get("plotting-options") == {}, "Check 2" settings.set("plotting-options", {"width": 400}) assert settings.get("plotting-options") == {"width": 400} settings.reset() assert settings.get("plotting-options") == {}, "Check 3" with pytest.raises(TypeError): settings.set("plotting-options", 3) settings.set("styles-directories", ["/a", "/b"]) assert settings.get("styles-directories") == ["/a", "/b"] settings.set("styles-directories", "/c", "/d") assert settings.get("styles-directories") == ["/c", "/d"] with pytest.raises(KeyError): settings.set("test", 42) with pytest.raises(KeyError): settings.get("test") with pytest.raises(ValueError): settings.set("url-download-timeout", "1M")
def test_unpack_zip(): TEST_URL = "https://get.ecmwf.int/test-data/climetlab/input/grib.zip" # Make sure we fail if not cached with pytest.raises(OfflineError), network_off(): ds = load_source("url", f"{TEST_URL}?time={time.time()}") with temp_directory() as tmpdir: with settings.temporary("cache-directory", tmpdir): ds = load_source("url", TEST_URL) assert len(ds) == 6, len(ds) with network_off(): # Check cache is used ds = load_source("url", TEST_URL) assert len(ds) == 6, len(ds) with pytest.raises(OfflineError), network_off(): # check force ds = load_source("url", TEST_URL, force=True) assert len(ds) == 6, len(ds) ds = load_source("url", TEST_URL, force=True) assert len(ds) == 6, len(ds) with network_off(): ds = load_source("url", TEST_URL) assert len(ds) == 6, len(ds) # Again ds = load_source("url", TEST_URL) assert len(ds) == 6, len(ds)
def mirror_dirs(): """Setup for mirror tests: - a temporary cache dir, - temporary settings, - two temporary directories """ with temp_directory() as cachedir: with settings.temporary(): settings.set("cache-directory", cachedir) with temp_directory() as mirrordir: with temp_directory() as mirrordir2: _reset_mirrors(use_env_var=False) yield mirrordir, mirrordir2
def test_cache_2(): with temp_directory() as tmpdir: with settings.temporary(): settings.set("cache-directory", tmpdir) settings.set("maximum-cache-size", "5MB") settings.set("number-of-download-threads", 5) assert cache_size() == 0 load_source( "url-pattern", "https://get.ecmwf.int/test-data/climetlab/1mb-{n}.bin", { "n": [0, 1, 2, 3, 4], }, ) assert cache_size() == 5 * 1024 * 1024, cache_size() cnt = 0 for i, f in enumerate(cache_entries()): print("FILE", i, f) cnt += 1 assert cnt == 5, f"Files in cache database: {cnt}" load_source( "url-pattern", "https://get.ecmwf.int/test-data/climetlab/1mb-{n}.bin", { "n": [5, 6, 7, 8, 9], }, ) assert cache_size( ) == 5 * 1024 * 1024, cache_size() / 1024.0 / 1024.0 cnt = 0 for i, f in enumerate(cache_entries()): print("FILE", i, f) cnt += 1 assert cnt == 5, f"Files in cache database: {cnt}" cnt = 0 for n in os.listdir(tmpdir): if n.startswith("cache-") and n.endswith(".db"): continue cnt += 1 assert cnt == 5, f"Files in cache directory: {cnt}"
def test_url_source_check_out_of_date(): def load(): load_source( "url", "http://download.ecmwf.int/test-data/metview/gallery/temp.bufr", ) with temp_directory() as tmpdir: with settings.temporary(): settings.set("cache-directory", tmpdir) load() settings.set("check-out-of-date-urls", False) with network_off(): load()
def test_temporary(): settings.reset() settings.set("styles-directories", "/c", "/d") settings.set("plotting-options", {"width": 400}) with settings.temporary("plotting-options", {"width": 100}): assert settings.get("styles-directories") == ["/c", "/d"] assert settings.get("plotting-options") == {"width": 100}, settings.get( "plotting-options" ) settings.set("plotting-options", {"width": 200}) assert settings.get("plotting-options") == {"width": 200} settings.reset() assert settings.get("plotting-options") == {} settings.set("plotting-options", {"width": 400}) settings.set("styles-directories", "/c", "/d") settings.reset() assert settings.get("plotting-options") == {}
def test_numbers(): with temp_directory() as tmpdir: with settings.temporary("cache-directory", tmpdir): settings.set("url-download-timeout", 30) assert settings.get("url-download-timeout") == 30 settings.set("url-download-timeout", "30") assert settings.get("url-download-timeout") == 30 settings.set("url-download-timeout", "30s") assert settings.get("url-download-timeout") == 30 settings.set("url-download-timeout", "2m") assert settings.get("url-download-timeout") == 120 settings.set("url-download-timeout", "10h") assert settings.get("url-download-timeout") == 36000 settings.set("url-download-timeout", "7d") assert settings.get("url-download-timeout") == 7 * 24 * 3600 with pytest.raises(ValueError): settings.set("url-download-timeout", "1x") settings.set("maximum-cache-size", "1") assert settings.get("maximum-cache-size") == 1 settings.set("maximum-cache-size", "1k") assert settings.get("maximum-cache-size") == 1024 settings.set("maximum-cache-size", "1kb") assert settings.get("maximum-cache-size") == 1024 settings.set("maximum-cache-size", "1k") assert settings.get("maximum-cache-size") == 1024 settings.set("maximum-cache-size", "1kb") assert settings.get("maximum-cache-size") == 1024 settings.set("maximum-cache-size", "1K") assert settings.get("maximum-cache-size") == 1024 settings.set("maximum-cache-size", "1M") assert settings.get("maximum-cache-size") == 1024 * 1024 settings.set("maximum-cache-size", "1G") assert settings.get("maximum-cache-size") == 1024 * 1024 * 1024 settings.set("maximum-cache-size", "1T") assert settings.get("maximum-cache-size") == 1024 * 1024 * 1024 * 1024 settings.set("maximum-cache-size", "1P") assert ( settings.get("maximum-cache-size") == 1024 * 1024 * 1024 * 1024 * 1024 ) settings.set("maximum-cache-size", None) assert settings.get("maximum-cache-size") is None settings.set("maximum-cache-disk-usage", "2%") assert settings.get("maximum-cache-disk-usage") == 2
def test_download_3(): with settings.temporary("download-out-of-date-urls", True): url = "https://get.ecmwf.int/test-data/climetlab/input/test.txt" download_and_cache(url)
def test_cache_4(): with settings.temporary(): settings.set("cache-directory", "/Volumes/RAMDisk/climetlab") settings.set("maximum-cache-disk-usage", "90%") for n in range(10): load_source("dummy-source", "zeros", size=100 * 1024 * 1024, n=n)
def test_cache_2(): with temp_directory() as tmpdir: with settings.temporary(): settings.set("cache-directory", tmpdir) settings.set("maximum-cache-size", "5MB") settings.set("number-of-download-threads", 5) assert cache_size() == 0 load_source( "url-pattern", f"{TEST_DATA_URL}/input/" + "1mb-{n}.bin", { "n": [0, 1, 2, 3, 4], }, ) cachesize = cache_size() expected = 5 * 1024 * 1024 if cachesize != expected: print(json.dumps(dump_cache_database(), indent=4)) assert cachesize == expected, ("before", cachesize / 1024.0 / 1024.0) cnt = 0 for i, f in enumerate(cache_entries()): # print("FILE", i, f) cnt += 1 if cnt != 5: print(json.dumps(dump_cache_database(), indent=4)) assert cnt == 5, f"Files in cache database (before): {cnt}" load_source( "url-pattern", f"{TEST_DATA_URL}/input/" + "1mb-{n}.bin", { "n": [5, 6, 7, 8, 9], }, ) cachesize = cache_size() expected = 5 * 1024 * 1024 if cachesize != expected: print(json.dumps(dump_cache_database(), indent=4)) assert cachesize == expected, ("after", cachesize / 1024.0 / 1024.0) cnt = 0 for i, f in enumerate(cache_entries()): LOG.debug("FILE %s %s", i, f) cnt += 1 if cnt != 5: print(json.dumps(dump_cache_database(), indent=4)) assert cnt == 5, f"Files in cache database (after): {cnt}" cnt = 0 for n in os.listdir(tmpdir): if n.startswith("cache-") and n.endswith(".db"): continue cnt += 1 if cnt != 5: print(json.dumps(dump_cache_database(), indent=4)) assert cnt == 5, f"Files in cache directory: {cnt}"