def test_chunks_region(self): with argopy.set_options(local_ftp=self.local_ftp): fetcher_args = {"src": self.src, "parallel": True, 'chunks': {'lon': 1, 'lat': 2, 'dpt': 1, 'time': 2}} for access_arg in self.requests["region"]: f = ArgoDataFetcher(**fetcher_args).region(access_arg) assert isinstance(f.to_xarray(), xr.Dataset) assert is_list_of_strings(f.fetcher.uri)
def test_nocache(self): with argopy.set_options(cachedir="dummy"): loader = ArgoDataFetcher(src=self.src, cache=False).profile(6902746, 34) loader.to_xarray() with pytest.raises(FileSystemHasNoCache): loader.fetcher.cachepath
def test_nocache(self): with argopy.set_options(cachedir="dummy", local_ftp=self.local_ftp): loader = ArgoDataFetcher(src=self.src, cache=False).profile(2901623, 12) loader.to_xarray() with pytest.raises(FileSystemHasNoCache): loader.fetcher.cachepath
def __test_region(self, bk): """ Test float for a given backend """ for arg in self.args["region"]: for mode in self.mode: f = ArgoDataFetcher(src=bk, mode=mode).region(arg) assert isinstance(f.to_xarray(), xr.Dataset) assert is_list_of_strings(f.uri)
def __testthis_profile(self, dataset): with argopy.set_options(local_ftp=self.local_ftp): fetcher_args = {"src": self.src, 'ds': dataset} for arg in self.args['profile']: f = ArgoDataFetcher(**fetcher_args).profile(*arg) assert isinstance(f.to_xarray(), xr.Dataset) assert is_list_of_strings(f.fetcher.uri)
def test_clearcache(self): with tempfile.TemporaryDirectory() as testcachedir: with argopy.set_options(cachedir=testcachedir, local_ftp=self.local_ftp): loader = ArgoDataFetcher(src=self.src, cache=True).float(2901623) loader.to_xarray() loader.clear_cache() with pytest.raises(CacheFileNotFound): loader.fetcher.cachepath
def __test_float(self, bk, **ftc_opts): """ Test float for a given backend """ for arg in self.args["float"]: for mode in self.mode: options = {**self.fetcher_opts, **ftc_opts} f = ArgoDataFetcher(src=bk, mode=mode, **options).float(arg) assert isinstance(f.to_xarray(), xr.Dataset) assert is_list_of_strings(f.uri)
def test_chunks_wmo(self): with argopy.set_options(local_ftp=self.local_ftp): fetcher_args = {"src": self.src, "parallel": True, "chunks_maxsize": {'wmo': 1}} for access_arg in self.requests["wmo"]: # f = ArgoDataFetcher(**fetcher_args).float(access_arg) f = ArgoDataFetcher(**fetcher_args).profile(access_arg, 1) assert isinstance(f.to_xarray(), xr.Dataset) assert is_list_of_strings(f.fetcher.uri) assert len(f.fetcher.uri) == len(access_arg)
def test_nocache(self): with tempfile.TemporaryDirectory() as testcachedir: with argopy.set_options(cachedir=testcachedir): loader = ArgoDataFetcher( src=self.src, cache=False).profile(*self.requests['profile'][0]) loader.to_xarray() with pytest.raises(FileSystemHasNoCache): loader.fetcher.cachepath
def test_caching_profile(self): with tempfile.TemporaryDirectory() as testcachedir: with argopy.set_options(cachedir=testcachedir, local_ftp=self.local_ftp): loader = ArgoDataFetcher(src=self.src, cache=True).profile(2901623, 1) # 1st call to load and save to cachedir: loader.to_xarray() # 2nd call to load from cached file ds = loader.to_xarray() assert isinstance(ds, xr.Dataset) assert is_list_of_strings(loader.fetcher.uri) assert is_list_of_strings(loader.fetcher.cachepath)
def test_chunks_wmo(self): for access_arg in self.requests["wmo"]: fetcher_args = { "src": self.src, "parallel": True, "chunks_maxsize": { "wmo": 1 }, } f = ArgoDataFetcher(**fetcher_args).profile(access_arg, 12).fetcher assert isinstance(f.to_xarray(), xr.Dataset) assert is_list_of_strings(f.uri) assert len(f.uri) == len(access_arg)
def test_clear_cache(): ftproot, flist = argopy.tutorial.open_dataset("localftp") with tempfile.TemporaryDirectory() as cachedir: with argopy.set_options(cachedir=cachedir, local_ftp=ftproot): loader = ArgoDataFetcher(src="localftp", cache=True).profile(2902696, 12) loader.to_xarray( ) # 1st call to load from source and save in memory loader.to_xarray( ) # 2nd call to load from memory and save in cache argopy.clear_cache() assert os.path.exists(cachedir) is True assert len(os.listdir(cachedir)) == 0
def test_caching_region(self): with tempfile.TemporaryDirectory() as testcachedir: with argopy.set_options(cachedir=testcachedir, local_ftp=self.local_ftp): loader = ArgoDataFetcher(src=self.src, cache=True).region( [-60, -40, 40., 60., 0., 100., '2007-08-01', '2007-09-01'] ) # 1st call to load and save to cachedir: loader.to_xarray() # 2nd call to load from cached file ds = loader.to_xarray() assert isinstance(ds, xr.Dataset) assert is_list_of_strings(loader.fetcher.uri) assert is_list_of_strings(loader.fetcher.cachepath)
def test_chunks_region(self): for access_arg in self.requests["region"]: fetcher_args = { "src": self.src, "parallel": True, "chunks": { "lon": 1, "lat": 2, "dpt": 1, "time": 2 }, } f = ArgoDataFetcher(**fetcher_args).region(access_arg).fetcher assert isinstance(f.to_xarray(), xr.Dataset) assert is_list_of_strings(f.uri) assert len(f.uri) == np.prod( [v for k, v in fetcher_args["chunks"].items()])
def test_caching_profile(self): with argopy.set_options(cachedir=self.testcachedir): loader = ArgoDataFetcher(src=self.src, cache=True).profile(6902746, 34) try: # 1st call to load from erddap and save to cachedir: ds = loader.to_xarray() # 2nd call to load from cached file ds = loader.to_xarray() assert isinstance(ds, xr.Dataset) assert isinstance(loader.fetcher.cachepath, str) shutil.rmtree(self.testcachedir) except ErddapServerError: # Test is passed when something goes wrong because of the erddap server, not our fault ! shutil.rmtree(self.testcachedir) pass except Exception: shutil.rmtree(self.testcachedir) raise
def main(): unsaved = [] for i in list_float: try: argo_loader = ArgoDataFetcher() #access certain data and profile ds = argo_loader.float(i).to_xarray() #conver to Panda df = ds.to_dataframe() with open(str(i) + '.csv', 'w') as f: df.to_csv(f, header=f.tell() == 0) print('> ' + str(i) + '.csv saved.') # some floats contain bad data and can not be saved except: print('> file could not be saved ' + str(i)) unsaved.append(i) pass print('Number of unsaved floats:', len(unsaved))
def test_clearcache(self): with tempfile.TemporaryDirectory() as testcachedir: with argopy.set_options(cachedir=testcachedir): loader = ArgoDataFetcher(src=self.src, cache=True).float( self.requests["float"][0]) loader.to_xarray( ) # 1st call to load from source and save in memory loader.to_xarray( ) # 2nd call to load from memory and save in cache loader.clear_cache() with pytest.raises(CacheFileNotFound): loader.fetcher.cachepath
# Accessing argo data by float using Argopy from argopy import DataFetcher as ArgoDataFetcher plt.style.use('default') argo_loader = ArgoDataFetcher() argo_loader = ArgoDataFetcher(backend='erddap') argo_loader = ArgoDataFetcher(cachedir='tmp') apDS=argo_loader.float(6901254).to_xarray() apDS2=argo_loader.profile(6901254,1).to_xarray() data=apDS2.argo.point2profile() data The core-Argo profile files contain the core parameters provided by a float: pressure, temperature, salinity, conductivity (PRES, TEMP, PSAL, CNDC). All additional parameters are managed in B-Argo data files (see ยง0). fig, ax = plt.subplots(1,2,figsize=(8,10)) #Temperature ax[0].plot(data.TEMP[0],-data.PRES[0],'ro',label='N_PROF=0 D Fetcher') ax[0].plot(data.TEMP[1],-data.PRES[1],'bo',label='N_PROF=1 A Fetcher') ax[0].plot(cy1D.TEMP[0],-cy1D.PRES[0],'k-',label='N_PROF=0 D') ax[0].set_title(cy1.TEMP.long_name) ax[0].set_ylabel(cy1.PRES.long_name) ax[0].grid() ax[0].legend(); ax[1].plot(data.PSAL[0],-data.PRES[0],'ro',label='N_PROF=0 D Fetcher') ax[1].plot(data.PSAL[1],-data.PRES[1],'bo',label='N_PROF=1 A Fetcher')
argo8 = ('5903135', dt.strptime('2015-10-15 17:40:00', time_format), -41.212, -56.244, 'NOc Antares', dt.strptime('2017-11-15 06:25:00', time_format)) buoys = [argo1, argo2, argo3, argo4, argo5, argo6, argo7, argo8] columns_details = [ 'argo_id', 'start_date', 'latitude', 'longitude', 'ship_plataform', 'last_date' ] argos_details = pd.DataFrame.from_records(buoys, columns=columns_details) ### Download data from ARGO platform argo_loader = ArgoDataFetcher() argo_teste = argo_loader.float(list_argos) df_argo = argo_teste.to_dataframe() df_argo_final = df_argo[[ 'CYCLE_NUMBER', 'DIRECTION', 'PLATFORM_NUMBER', 'PRES', 'PSAL', 'TEMP', 'TIME', 'LONGITUDE', 'LATITUDE' ]] df_argo_final.loc[:, ['PSAL', 'TEMP']] = df_argo_final[[ 'PSAL', 'TEMP' ]].apply(lambda x: pd.Series.round(x, 3)) ### INSERTING DATA ON database
def test_bar_plot_profile(self): with argopy.set_options(local_ftp=self.local_ftp): for arg in self.requests["profile"]: loader = ArgoDataFetcher(src=self.src).profile(*arg) self.__test_bar_plot(loader.to_index(full=True))