def test_nocache(self):
with argopy.set_options(cachedir="dummy"):
loader = ArgoDataFetcher(src=self.src,
cache=False).profile(6902746, 34)
loader.to_xarray()
with pytest.raises(FileSystemHasNoCache):
loader.fetcher.cachepath
def __testthis_profile(self, dataset):
with argopy.set_options(local_ftp=self.local_ftp):
fetcher_args = {"src": self.src, 'ds': dataset}
for arg in self.args['profile']:
f = ArgoDataFetcher(**fetcher_args).profile(*arg)
assert isinstance(f.to_xarray(), xr.Dataset)
assert is_list_of_strings(f.fetcher.uri)
def test_nocache(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir, local_ftp=self.local_ftp):
loader = ArgoDataFetcher(src=self.src, cache=False).profile(2901623, 12)
loader.to_xarray()
with pytest.raises(FileSystemHasNoCache):
loader.fetcher.cachepath
def test_chunks_region(self):
with argopy.set_options(local_ftp=self.local_ftp):
fetcher_args = {"src": self.src, "parallel": True, 'chunks': {'lon': 1, 'lat': 2, 'dpt': 1, 'time': 2}}
for access_arg in self.requests["region"]:
f = ArgoDataFetcher(**fetcher_args).region(access_arg)
assert isinstance(f.to_xarray(), xr.Dataset)
assert is_list_of_strings(f.fetcher.uri)
def test_nocache(self):
with argopy.set_options(cachedir="dummy", local_ftp=self.local_ftp):
loader = ArgoIndexFetcher(src=self.src,
cache=False).profile(2901623, 2)
loader.to_xarray()
with pytest.raises(FileSystemHasNoCache):
loader.fetcher.cachepath
def test_cachepath_notfound(self):
with argopy.set_options(cachedir=self.testcachedir):
loader = ArgoDataFetcher(src=self.src,
cache=True).profile(6902746, 34)
with pytest.raises(CacheFileNotFound):
loader.fetcher.cachepath
shutil.rmtree(self.testcachedir) # Make sure the cache is left empty
def test_nocache(self):
with argopy.set_options(cachedir=self.testcachedir):
loader = ArgoIndexFetcher(src='erddap', cache=False).float(6902746)
loader.to_xarray()
with pytest.raises(FileSystemHasNoCache):
loader.fetcher.cachepath
shutil.rmtree(self.testcachedir) # Make sure the cache is empty
def test_clearcache(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir, local_ftp=self.local_ftp):
loader = ArgoDataFetcher(src=self.src, cache=True).float(2901623)
loader.to_xarray()
loader.clear_cache()
with pytest.raises(CacheFileNotFound):
loader.fetcher.cachepath
def test_cachepath_notfound(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir):
loader = ArgoDataFetcher(
src=self.src,
cache=True).profile(*self.requests['profile'][0])
with pytest.raises(CacheFileNotFound):
loader.fetcher.cachepath
def test_nocache(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir):
loader = ArgoIndexFetcher(src=self.src, cache=False).float(
self.requests['float'][0])
loader.to_xarray()
with pytest.raises(FileSystemHasNoCache):
loader.fetcher.cachepath
def test_clearcache(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir):
loader = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0])
loader.to_xarray()
loader.clear_cache()
with pytest.raises(CacheFileNotFound):
loader.fetcher.cachepath
def test_clear_cache():
# Fetch data to cache:
ftproot, flist = argopy.tutorial.open_dataset('localftp')
testcachedir = os.path.expanduser(os.path.join("~", ".argopytest_tmp"))
with argopy.set_options(cachedir=testcachedir, local_ftp=ftproot):
ArgoDataFetcher(src='localftp').profile(2902696, 12).to_xarray()
# Then clean it:
argopy.clear_cache()
assert os.path.isdir(testcachedir) is False
def test_cachepath_notfound(self):
testcachedir = os.path.expanduser(os.path.join("~", ".argopytest_tmp"))
with argopy.set_options(cachedir=testcachedir,
local_ftp=self.local_ftp):
loader = ArgoIndexFetcher(src=self.src,
cache=True).profile(2901623, 2)
with pytest.raises(CacheFileNotFound):
loader.fetcher.cachepath
shutil.rmtree(testcachedir) # Make sure the cache folder is cleaned
def test_chunks_wmo(self):
with argopy.set_options(local_ftp=self.local_ftp):
fetcher_args = {"src": self.src, "parallel": True, "chunks_maxsize": {'wmo': 1}}
for access_arg in self.requests["wmo"]:
# f = ArgoDataFetcher(**fetcher_args).float(access_arg)
f = ArgoDataFetcher(**fetcher_args).profile(access_arg, 1)
assert isinstance(f.to_xarray(), xr.Dataset)
assert is_list_of_strings(f.fetcher.uri)
assert len(f.fetcher.uri) == len(access_arg)
def test_clear_cache():
ftproot, flist = argopy.tutorial.open_dataset("localftp")
with tempfile.TemporaryDirectory() as cachedir:
with argopy.set_options(cachedir=cachedir, local_ftp=ftproot):
ArgoDataFetcher(src="localftp").profile(2902696, 12).to_xarray()
ArgoDataFetcher(src="localftp").profile(2902696, 13).to_xarray()
argopy.clear_cache()
assert os.path.exists(cachedir) is True
assert len(os.listdir(cachedir)) == 0
def __testthis(self, dataset):
for access_point in self.args:
if access_point == 'profile':
for arg in self.args['profile']:
with argopy.set_options(local_ftp=self.local_ftp):
try:
ds = ArgoIndexFetcher(src=self.src).profile(
*arg).to_xarray()
assert isinstance(ds, xr.Dataset)
except Exception:
print(
"ERROR LOCALFTP request:\n",
ArgoIndexFetcher(src=self.src).profile(
*arg).fetcher.cname())
pass
if access_point == 'float':
for arg in self.args['float']:
with argopy.set_options(local_ftp=self.local_ftp):
try:
ds = ArgoIndexFetcher(
src=self.src).float(arg).to_xarray()
assert isinstance(ds, xr.Dataset)
except Exception:
print(
"ERROR LOCALFTP request:\n",
ArgoIndexFetcher(
src=self.src).float(arg).fetcher.cname())
pass
if access_point == 'region':
for arg in self.args['region']:
with argopy.set_options(local_ftp=self.local_ftp):
try:
ds = ArgoIndexFetcher(
src=self.src).region(arg).to_xarray()
assert isinstance(ds, xr.Dataset)
except Exception:
print(
"ERROR LOCALFTP request:\n",
ArgoIndexFetcher(
src=self.src).region(arg).fetcher.cname())
pass
def test_caching(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir):
loader = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0])
# 1st call to load and save to cache:
loader.to_xarray()
# 2nd call to load from cached file:
ds = loader.to_xarray()
assert isinstance(ds, xr.Dataset)
assert isinstance(loader.fetcher.cachepath, str)
def test_nocache(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir,
local_ftp=self.local_ftp):
loader = ArgoIndexFetcher(
src=self.src,
cache=False).profile(*self.requests['profile'][0])
loader.to_dataframe()
with pytest.raises(FileSystemHasNoCache):
loader.fetcher.cachepath
def test_methods(self):
args_list = [
{"src": self.src, "parallel": "thread"},
{"src": self.src, "parallel": True, "parallel_method": "thread"},
{"src": self.src, "parallel": "process"},
{"src": self.src, "parallel": True, "parallel_method": "process"},
]
with argopy.set_options(local_ftp=self.local_ftp):
for fetcher_args in args_list:
loader = ArgoDataFetcher(**fetcher_args).float(self.requests["wmo"][0])
assert isinstance(loader, argopy.fetchers.ArgoDataFetcher)
args_list = [
{"src": self.src, "parallel": "toto"},
{"src": self.src, "parallel": True, "parallel_method": "toto"},
]
with argopy.set_options(local_ftp=self.local_ftp):
for fetcher_args in args_list:
with pytest.raises(ValueError):
ArgoDataFetcher(**fetcher_args).float(self.requests["wmo"][0])
def test_caching_profile(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir, local_ftp=self.local_ftp):
loader = ArgoDataFetcher(src=self.src, cache=True).profile(2901623, 1)
# 1st call to load and save to cachedir:
loader.to_xarray()
# 2nd call to load from cached file
ds = loader.to_xarray()
assert isinstance(ds, xr.Dataset)
assert is_list_of_strings(loader.fetcher.uri)
assert is_list_of_strings(loader.fetcher.cachepath)
def test_clearcache(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir):
loader = ArgoDataFetcher(src=self.src, cache=True).float(
self.requests["float"][0])
loader.to_xarray(
) # 1st call to load from source and save in memory
loader.to_xarray(
) # 2nd call to load from memory and save in cache
loader.clear_cache()
with pytest.raises(CacheFileNotFound):
loader.fetcher.cachepath
def test_caching(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir,
local_ftp=self.local_ftp):
loader = ArgoIndexFetcher(src=self.src, cache=True).float(
self.requests['float'][0])
# 1st call to load and save to cache:
loader.to_dataframe()
# 2nd call to load from cached file:
df = loader.to_dataframe()
assert isinstance(df, pd.core.frame.DataFrame)
assert isinstance(loader.fetcher.cachepath, str)
def test_caching_region(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir):
fetcher = (ArgoDataFetcher(src=self.src, cache=True).region(
self.requests['region'][1]).fetcher)
# 1st call to load and save to cachedir:
fetcher.to_xarray()
# 2nd call to load from cached file
ds = fetcher.to_xarray()
assert isinstance(ds, xr.Dataset)
assert is_list_of_strings(fetcher.uri)
assert is_list_of_strings(fetcher.cachepath)
def test_caching_region(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir, local_ftp=self.local_ftp):
loader = ArgoDataFetcher(src=self.src, cache=True).region(
[-60, -40, 40., 60., 0., 100., '2007-08-01', '2007-09-01']
)
# 1st call to load and save to cachedir:
loader.to_xarray()
# 2nd call to load from cached file
ds = loader.to_xarray()
assert isinstance(ds, xr.Dataset)
assert is_list_of_strings(loader.fetcher.uri)
assert is_list_of_strings(loader.fetcher.cachepath)
def test_clear_cache():
ftproot, flist = argopy.tutorial.open_dataset("localftp")
with tempfile.TemporaryDirectory() as cachedir:
with argopy.set_options(cachedir=cachedir, local_ftp=ftproot):
loader = ArgoDataFetcher(src="localftp",
cache=True).profile(2902696, 12)
loader.to_xarray(
) # 1st call to load from source and save in memory
loader.to_xarray(
) # 2nd call to load from memory and save in cache
argopy.clear_cache()
assert os.path.exists(cachedir) is True
assert len(os.listdir(cachedir)) == 0
def __testthis(self):
for access_point in self.args:
if access_point == "profile":
for arg in self.args["profile"]:
with argopy.set_options(local_ftp=self.local_ftp):
fetcher = ArgoIndexFetcher(src=self.src).profile(*arg).fetcher
ds = fetcher.to_xarray()
assert isinstance(ds, xr.Dataset)
if access_point == "float":
for arg in self.args["float"]:
with argopy.set_options(local_ftp=self.local_ftp):
fetcher = ArgoIndexFetcher(src=self.src).float(arg).fetcher
ds = fetcher.to_xarray()
assert isinstance(ds, xr.Dataset)
if access_point == "region":
for arg in self.args["region"]:
with argopy.set_options(local_ftp=self.local_ftp):
fetcher = ArgoIndexFetcher(src=self.src).region(arg).fetcher
ds = fetcher.to_xarray()
assert isinstance(ds, xr.Dataset)
def test_clearcache(self):
with tempfile.TemporaryDirectory() as testcachedir:
with argopy.set_options(cachedir=testcachedir,
local_ftp=self.local_ftp):
loader = ArgoIndexFetcher(
src=self.src,
cache=True).profile(*self.requests['profile'][0])
loader.to_dataframe(
) # 1st call to load from source and save in memory
loader.to_dataframe(
) # 2nd call to load from memory and save in cache
loader.clear_cache()
with pytest.raises(CacheFileNotFound):
loader.fetcher.cachepath
def __testthis(self, dataset):
ftproot, flist = argopy.tutorial.open_dataset('localftp')
self.local_ftp = os.path.join(ftproot, 'dac')
for access_point in self.args:
if access_point == 'profile':
for arg in self.args['profile']:
with argopy.set_options(local_ftp=self.local_ftp):
try:
ds = ArgoDataFetcher(
src='localftp',
ds=dataset).profile(*arg).to_xarray()
assert isinstance(ds, xr.Dataset) == True
except:
print(
"LOCALFTP request:\n",
ArgoDataFetcher(
src='localftp',
ds=dataset).profile(*arg).fetcher.files)
pass
if access_point == 'float':
for arg in self.args['float']:
with argopy.set_options(local_ftp=self.local_ftp):
try:
ds = ArgoDataFetcher(
src='localftp',
ds=dataset).float(arg).to_xarray()
assert isinstance(ds, xr.Dataset) == True
except:
print(
"LOCALFTP request:\n",
ArgoDataFetcher(
src='localftp',
ds=dataset).float(arg).fetcher.files)
pass
def test_load(self):
with argopy.set_options(local_ftp=self.local_ftp):
f, fetcher = self.__get_fetcher(pt='float')
fetcher.load()
assert is_list_of_strings(fetcher.uri)
assert isinstance(fetcher.data, xr.Dataset)
assert isinstance(fetcher.index, pd.core.frame.DataFrame)
# Change the access point:
new_fetcher = f.profile(fetcher._AccessPoint_data['wmo'], 1)
new_fetcher.load()
assert is_list_of_strings(new_fetcher.uri)
assert isinstance(new_fetcher.data, xr.Dataset)
assert isinstance(new_fetcher.index, pd.core.frame.DataFrame)
def __testthis_region(self, dataset):
with argopy.set_options(local_ftp=self.local_ftp):
for arg in self.args['region']:
try:
ds = ArgoDataFetcher(src=self.src,
ds=dataset).region(arg).to_xarray()
assert isinstance(ds, xr.Dataset)
except ErddapServerError:
# Test is passed when something goes wrong because of the erddap server, not our fault !
pass
except Exception:
print(
"ERROR LOCALFTP request:\n",
ArgoDataFetcher(src=self.src,
ds=dataset).region(arg).fetcher.files)
pass