def getSession(self, dataSource: DataSource) -> Session: session: Session = None if dataSource.auth == "esgf": from pydap.cas.esgf import setup_session openid = EdasEnv.get("esgf.openid", "") password = EdasEnv.get("esgf.password", "") username = EdasEnv.get("esgf.username", openid.split("/")[-1]) session = setup_session(openid, password, username, check_url=dataSource.address) elif dataSource.auth == "urs": from pydap.cas.urs import setup_session username = EdasEnv.get("urs.username", "") password = EdasEnv.get("urs.password", "") session = setup_session(username, password, check_url=dataSource.address) elif dataSource.auth == "cookie": from pydap.cas.get_cookies import setup_session username = EdasEnv.get("auth.username", "") password = EdasEnv.get("auth.password", "") auth_url = EdasEnv.get("auth.url", "") session = setup_session(auth_url, username, password) elif dataSource.auth is not None: raise Exception("Unknown authentication method: " + dataSource.auth) return session
def test_registration_esgf_auth(): """ Attempt to access a ESGF OPENDAP link for which the user has not yet selected a registration group. This procedure has to be completed only once per project over the lifetime of an ESGF OPENID Requires OPENID_ESGF_NO_REG and PASSWORD_ESGF_NO_REG environment variables. These must be associated with credentials where no group was selected. """ with pytest.raises(UserWarning): esgf.setup_session(os.environ.get('OPENID_ESGF_NO_REG'), os.environ.get('PASSWORD_ESGF_NO_REG'), check_url=url)
def test_variable_esgf_query(self): assert (os.environ.get('OPENID_ESGF')) assert (os.environ.get('PASSWORD_ESGF')) session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=self.url) # Ensure authentication: res = pydap.net.follow_redirect(self.test_url, session=session) assert (res.status_code == 200) dataset = open_url(self.url, session=session, output_grid=False) data = dataset['pr'][0, 200:205, 100:105] expected_data = [[[ 5.23546005e-05, 5.48864300e-05, 5.23546005e-05, 6.23914966e-05, 6.26627589e-05 ], [ 5.45247385e-05, 5.67853021e-05, 5.90458621e-05, 6.51041701e-05, 6.23914966e-05 ], [ 5.57906533e-05, 5.84129048e-05, 6.37478297e-05, 5.99500854e-05, 5.85033267e-05 ], [ 5.44343166e-05, 5.45247385e-05, 5.60619228e-05, 5.58810752e-05, 4.91898136e-05 ], [ 5.09982638e-05, 4.77430549e-05, 4.97323490e-05, 5.43438946e-05, 5.26258664e-05 ]]] assert (np.isclose(data, expected_data).all())
def test_variable_esgf_query(self): session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=self.url) dataset = open_url(self.url, session=session) data = dataset['pr'][0, 200:205, 100:105] expected_data = [[[ 5.23546005e-05, 5.48864300e-05, 5.23546005e-05, 6.23914966e-05, 6.26627589e-05 ], [ 5.45247385e-05, 5.67853021e-05, 5.90458621e-05, 6.51041701e-05, 6.23914966e-05 ], [ 5.57906533e-05, 5.84129048e-05, 6.37478297e-05, 5.99500854e-05, 5.85033267e-05 ], [ 5.44343166e-05, 5.45247385e-05, 5.60619228e-05, 5.58810752e-05, 4.91898136e-05 ], [ 5.09982638e-05, 4.77430549e-05, 4.97323490e-05, 5.43438946e-05, 5.26258664e-05 ]]] assert (np.isclose(data, expected_data).all())
def test_variable_esgf_query(self): assert(os.environ.get('OPENID_ESGF')) assert(os.environ.get('PASSWORD_ESGF')) session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=self.url) # Ensure authentication: res = pydap.net.follow_redirect(self.test_url, session=session) assert(res.status_code == 200) dataset = open_url(self.url, session=session, output_grid=False) data = dataset['pr'][0, 200:205, 100:105] expected_data = [[[5.23546005e-05, 5.48864300e-05, 5.23546005e-05, 6.23914966e-05, 6.26627589e-05], [5.45247385e-05, 5.67853021e-05, 5.90458621e-05, 6.51041701e-05, 6.23914966e-05], [5.57906533e-05, 5.84129048e-05, 6.37478297e-05, 5.99500854e-05, 5.85033267e-05], [5.44343166e-05, 5.45247385e-05, 5.60619228e-05, 5.58810752e-05, 4.91898136e-05], [5.09982638e-05, 4.77430549e-05, 4.97323490e-05, 5.43438946e-05, 5.26258664e-05]]] assert(np.isclose(data, expected_data).all())
def test_dimension_esgf_query(self): session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=self.url) dataset = open_url(self.url, session=session) data = dataset['time'][:10] expected_data = np.array([ 16832.5, 16833.5, 16834.5, 16835.5, 16836.5, 16837.5, 16838.5, 16839.5, 16840.5, 16841.5 ]) assert (np.isclose(data, expected_data).all())
def test_variable_esgf_query(): session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=url) # Ensure authentication: res = pydap.net.follow_redirect(test_url, session=session) assert(res.status_code == 200) dataset = open_url(url, session=session, output_grid=False) data = dataset['orog'][103:105, 100:102] expected_data = [[271.36645508, 166.85339355], [304.22286987, 178.85267639]] assert(np.isclose(data, expected_data).all())
def test_dimension_esgf_query(): session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=url) # Ensure authentication: res = pydap.net.follow_redirect(test_url, session=session) assert(res.status_code == 200) dataset = open_url(url, session=session) data = dataset['lon'][:5] expected_data = np.array([0., 1.25, 2.5, 3.75, 5.]) assert(np.isclose(data, expected_data).all())
def test_variable_esgf_query(): session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=url) # Ensure authentication: res = pydap.net.follow_redirect(test_url, session=session) assert (res.status_code == 200) dataset = open_url(url, session=session, output_grid=False) data = dataset['orog'][103:105, 100:102] expected_data = [[271.36645508, 166.85339355], [304.22286987, 178.85267639]] assert (np.isclose(data, expected_data).all())
def test_dimension_esgf_query(): session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=url) # Ensure authentication: res = pydap.net.follow_redirect(test_url, session=session) assert(res.status_code == 200) dataset = open_url(url, session=session) data = dataset['lon'][:5] expected_data = np.array([0.0, 2.8125, 5.625, 8.4375, 11.25]) assert(np.isclose(data, expected_data).all())
def test_variable_esgf_query(): session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=url) # Ensure authentication: res = pydap.net.follow_redirect(test_url, session=session) assert(res.status_code == 200) dataset = open_url(url, session=session, output_grid=False) data = dataset['orog'][50:55, 50:55] expected_data = [[197.70425, 16.319595, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [677.014, 628.29675, 551.06, 455.5758, 343.7354], [1268.3304, 1287.9553, 1161.0402, 978.3153, 809.143]] assert(np.isclose(data, expected_data).all())
def test_dimension_esgf_query(self): assert(os.environ.get('OPENID_ESGF')) assert(os.environ.get('PASSWORD_ESGF')) session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=self.url) # Ensure authentication: res = pydap.net.follow_redirect(self.test_url, session=session) assert(res.status_code == 200) dataset = open_url(self.url, session=session) data = dataset['time'][:10] expected_data = np.array([16832.5, 16833.5, 16834.5, 16835.5, 16836.5, 16837.5, 16838.5, 16839.5, 16840.5, 16841.5]) assert(np.isclose(data, expected_data).all())
def _get_session(self): if self.auth is None: session = None else: if self.auth == "esgf": from pydap.cas.esgf import setup_session elif self.auth == "urs": from pydap.cas.urs import setup_session else: raise ValueError( "Authentication method should either be None, 'esgf' or 'urs', " f"got '{self.auth}' instead." ) username = os.getenv('DAP_USER', None) password = os.getenv('DAP_PASSWORD', None) session = setup_session(username, password, check_url=self.urlpath) return session
def test_dimension_esgf_query(self): assert (os.environ.get('OPENID_ESGF')) assert (os.environ.get('PASSWORD_ESGF')) session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=self.url) # Ensure authentication: res = pydap.net.follow_redirect(self.test_url, session=session) assert (res.status_code == 200) dataset = open_url(self.url, session=session) data = dataset['time'][:10] expected_data = np.array([ 16832.5, 16833.5, 16834.5, 16835.5, 16836.5, 16837.5, 16838.5, 16839.5, 16840.5, 16841.5 ]) assert (np.isclose(data, expected_data).all())
def test_basic_esgf_auth(): """ Set up PyDAP to use the ESGF request() function. The intent here is to ensure that pydap.net is able to open and url if and only if requests is able to open the same url. """ session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=url) res = requests.get(test_url, cookies=session.cookies) assert (res.status_code == 200) res.close() res = pydap.net.follow_redirect(test_url, session=session) assert (res.status_code == 200)
def test_basic_esgf_auth(): """ Set up PyDAP to use the ESGF request() function. The intent here is to ensure that pydap.net is able to open and url if and only if requests is able to open the same url. """ session = esgf.setup_session(os.environ.get('OPENID_ESGF'), os.environ.get('PASSWORD_ESGF'), check_url=url) res = requests.get(test_url, cookies=session.cookies) assert(res.status_code == 200) res.close() res = pydap.net.follow_redirect(test_url, session=session) assert(res.status_code == 200)
def DownloadPoints(variable, latitudes_s, longitudes_s, url, outfile, session=None): """ Download time-series for a list of lats/longs from a single opendap dataset (ie CORDEX netcdf). The output nc file has primary dimensions location and time. Inputs: ------- variable - string. variable to download, eg 'tas' or 'pr' latitudes_s, longitudes_s are python lists url - opendap url outfile - string. where to write the file. session - Note or pydap.cas.esgf session. None for a new session. Might save a little time if you reuse sessions rather than creating new ones for each download? Might be more trouble than it is worth, seems to get time-outs with re-using sesions. Best to just leave as None. But if you want to try: session = setup_session(config.openid, config.password, verify=True, check_url=url) """ if session == None: session = setup_session(config.openid, config.password, verify=True, check_url=url) d = open_url(url, session=session) # lat/long to grid mapping Message('download from %s' % url, 2) #### Message(' ... reading header and extracting pixel indices...', 2, newline=False) lat = d['lat'].array.data[:, :] lon = d['lon'].array.data[:, :] for i, l in enumerate(longitudes_s): if l > 180: longitudes_s[i] = l - 360 lon_shift = lon > 180 lon[lon_shift] = lon[lon_shift] - 360 ns = len(longitudes_s) rlat_idx = [None] * ns rlon_idx = [None] * ns lat_of_pixel = np.zeros(ns, dtype=lat.dtype) lon_of_pixel = np.zeros(ns, dtype=lat.dtype) Message('done.', 2) #### Message(' ... reading and processing header for %d locations...' % ns, 2, False) for i in range(ns): longitude = longitudes_s[i] latitude = latitudes_s[i] dist = ((abs(lon - longitude)**2) + (abs(lat - latitude)**2)) mask = dist == np.amin(dist) lat_of_pixel[i] = lat[mask] lon_of_pixel[i] = lon[mask] rlat_idx[i] = np.asscalar(mask.nonzero()[0]) rlon_idx[i] = np.asscalar(mask.nonzero()[1]) Message('done.', 2) #### # open a new netCDF file for writing, and download data and write it in directly! Message(' ... downloading and writing to %s...' % outfile, 2) with Dataset(outfile, 'w', format='NETCDF4') as ncfile: for name in d.attributes['NC_GLOBAL']: attr_value = d.attributes['NC_GLOBAL'][name] if name[0] != '_' and isinstance(attr_value, str): ncfile.setncattr(name, attr_value) # create the x and y dimensions. ncfile.createDimension('location', ns) ncfile.createDimension('time', None) times = ncfile.createVariable('time', d['time'].dtype.name, ('time', )) latitudes = ncfile.createVariable('lat', lat_of_pixel.dtype.name, ('location', )) longitudes = ncfile.createVariable('lon', lon_of_pixel.dtype.name, ('location', )) M = ncfile.createVariable(variable, d[variable].array.dtype.name, ( 'time', 'location', )) latitudes[:] = lat_of_pixel longitudes[:] = lon_of_pixel times[:] = d['time'].data[:] for i in range(ns): Message(' ... station %d of %d' % (i, ns), 2) M[:, i] = d[variable].array.data[:, rlat_idx[i], rlon_idx[i]] for cvar in ( 'time', variable, ): for name in d[cvar].attributes: attr_value = d[cvar].attributes[name] if name[0] != '_' and isinstance(attr_value, str): ncfile.variables[cvar].setncattr(name, attr_value) Message('done.', 2)
def DownloadSlice(variable, url, outfile, rlat_idx=None, rlon_idx=None, time_idx=None, session=None): """ Download a slice for a single opendap dataset (ie CORDEX netcdf). Inputs: ------- variable - string. variable to download, eg 'tas' or 'pr' url - opendap url. outfile - string. where to write the file. rlat_idx, rlon_idx, time_idx - slices for downloading a subset of the grids. eg: rlat_idx=slice(250,391) rlon_idx=slice(198,286) None for the entire axis (all that is supported for time??) session - Note or pydap.cas.esgf session. None for a new session. Might save a little time if you reuse sessions rather than creating new ones for each download? Might be more trouble than it is worth, seems to get time-outs with re-using sesions. Best to just leave as None. But if you want to try: session = setup_session(config.openid, config.password, verify=True, check_url=url) """ if session == None: session = setup_session(config.openid, config.password, verify=True, check_url=config.check_url) Message('download from %s' % url, 2) #### # defaults for all! presume this actually works... if rlat_idx == None: rlat_idx = slice(0, None) if rlon_idx == None: rlon_idx = slice(0, None) if time_idx == None: time_idx = slice(0, None) #### d = open_url(url, session=session) template = d['lat'].array.data[rlat_idx, rlon_idx] # open a new netCDF file for writing, and download data and write it in directly! Message(' ... downloading and writing to %s...' % outfile, 2) #with Dataset(outfile,'w', format='NETCDF4') as ncfile: ncfile = Dataset(outfile, 'w', format='NETCDF4') for name in d.attributes['NC_GLOBAL']: attr_value = d.attributes['NC_GLOBAL'][name] if name[0] != '_' and isinstance(attr_value, str): ncfile.setncattr(name, attr_value) # create the x and y dimensions. ncfile.createDimension('rlon', template.shape[1]) ncfile.createDimension('rlat', template.shape[0]) ncfile.createDimension('time', None) times = ncfile.createVariable('time', d['time'].dtype.name, ('time', )) rlats = ncfile.createVariable('rlat', d['rlat'].dtype.name, ('rlat', )) rlons = ncfile.createVariable('rlon', d['rlat'].dtype.name, ('rlon', )) latitudes = ncfile.createVariable('lat', d['lat'].array.dtype.name, ( 'rlat', 'rlon', )) longitudes = ncfile.createVariable('lon', d['lat'].array.dtype.name, ( 'rlat', 'rlon', )) M = ncfile.createVariable(variable, d[variable].array.dtype.name, ( 'time', 'rlat', 'rlon', )) rlats[:] = d['rlat'].data[rlat_idx] rlons[:] = d['rlon'].data[rlon_idx] latitudes[:, :] = d['lat'].array.data[rlat_idx, rlon_idx] longitudes[:, :] = d['lon'].array.data[rlat_idx, rlon_idx] times[:] = d['time'].data[:] M[:, :, :] = d[variable].array.data[time_idx, rlat_idx, rlon_idx] for cvar in ('time', variable, 'rlat', 'rlon', 'lat', 'lon'): for name in d[cvar].attributes: attr_value = d[cvar].attributes[name] if name[0] != '_' and isinstance(attr_value, str): ncfile.variables[cvar].setncattr(name, attr_value) ncfile.close() Message('done.', 2)
def _get_session(self): from pydap.cas.esgf import setup_session username = os.getenv('DAP_USER', None) password = os.getenv('DAP_PASSWORD', None) return setup_session(username, password, check_url=self.urlpath)
""" Example of using CORDEX_DownloadFunctions to download a sub-grid of CORDEX data """ import os #from pydap.client import open_url from pydap.cas.esgf import setup_session from CORDEX_DownloadFunctions import JSONParser, ConcatenatedCatalogSlice import config # this is just to check that things are working... session = setup_session(config.openid, config.password, verify=True, check_url=config.check_url) # ~Norden in EU44 #rlat_idx=slice(61,101) #rlon_idx=slice(48,74) # ~Norden in EU11 rlat_idx = slice(250, 391) rlon_idx = slice(198, 286) ModelSelectionFilename = os.path.join('CORDEX_Example', 'monthly_test.json') Outdir = os.path.join('CORDEX_Example', 'Data') p = JSONParser(ModelSelectionFilename) ConcatenatedCatalogSlice(p.Catalog,