def getSession(self, dataSource: DataSource) -> Session:
session: Session = None
if dataSource.auth == "esgf":
from pydap.cas.esgf import setup_session
openid = EdasEnv.get("esgf.openid", "")
password = EdasEnv.get("esgf.password", "")
username = EdasEnv.get("esgf.username", openid.split("/")[-1])
session = setup_session(openid,
password,
username,
check_url=dataSource.address)
elif dataSource.auth == "urs":
from pydap.cas.urs import setup_session
username = EdasEnv.get("urs.username", "")
password = EdasEnv.get("urs.password", "")
session = setup_session(username,
password,
check_url=dataSource.address)
elif dataSource.auth == "cookie":
from pydap.cas.get_cookies import setup_session
username = EdasEnv.get("auth.username", "")
password = EdasEnv.get("auth.password", "")
auth_url = EdasEnv.get("auth.url", "")
session = setup_session(auth_url, username, password)
elif dataSource.auth is not None:
raise Exception("Unknown authentication method: " +
dataSource.auth)
return session
def test_registration_esgf_auth():
"""
Attempt to access a ESGF OPENDAP link for which
the user has not yet selected a registration group.
This procedure has to be completed only once per project
over the lifetime of an ESGF OPENID
Requires OPENID_ESGF_NO_REG and PASSWORD_ESGF_NO_REG
environment variables. These must be associated with credentials
where no group was selected.
"""
with pytest.raises(UserWarning):
esgf.setup_session(os.environ.get('OPENID_ESGF_NO_REG'),
os.environ.get('PASSWORD_ESGF_NO_REG'),
check_url=url)
def test_registration_esgf_auth():
"""
Attempt to access a ESGF OPENDAP link for which
the user has not yet selected a registration group.
This procedure has to be completed only once per project
over the lifetime of an ESGF OPENID
Requires OPENID_ESGF_NO_REG and PASSWORD_ESGF_NO_REG
environment variables. These must be associated with credentials
where no group was selected.
"""
with pytest.raises(UserWarning):
esgf.setup_session(os.environ.get('OPENID_ESGF_NO_REG'),
os.environ.get('PASSWORD_ESGF_NO_REG'),
check_url=url)
def test_variable_esgf_query(self):
assert (os.environ.get('OPENID_ESGF'))
assert (os.environ.get('PASSWORD_ESGF'))
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=self.url)
# Ensure authentication:
res = pydap.net.follow_redirect(self.test_url, session=session)
assert (res.status_code == 200)
dataset = open_url(self.url, session=session, output_grid=False)
data = dataset['pr'][0, 200:205, 100:105]
expected_data = [[[
5.23546005e-05, 5.48864300e-05, 5.23546005e-05, 6.23914966e-05,
6.26627589e-05
],
[
5.45247385e-05, 5.67853021e-05, 5.90458621e-05,
6.51041701e-05, 6.23914966e-05
],
[
5.57906533e-05, 5.84129048e-05, 6.37478297e-05,
5.99500854e-05, 5.85033267e-05
],
[
5.44343166e-05, 5.45247385e-05, 5.60619228e-05,
5.58810752e-05, 4.91898136e-05
],
[
5.09982638e-05, 4.77430549e-05, 4.97323490e-05,
5.43438946e-05, 5.26258664e-05
]]]
assert (np.isclose(data, expected_data).all())
def test_variable_esgf_query(self):
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=self.url)
dataset = open_url(self.url, session=session)
data = dataset['pr'][0, 200:205, 100:105]
expected_data = [[[
5.23546005e-05, 5.48864300e-05, 5.23546005e-05, 6.23914966e-05,
6.26627589e-05
],
[
5.45247385e-05, 5.67853021e-05, 5.90458621e-05,
6.51041701e-05, 6.23914966e-05
],
[
5.57906533e-05, 5.84129048e-05, 6.37478297e-05,
5.99500854e-05, 5.85033267e-05
],
[
5.44343166e-05, 5.45247385e-05, 5.60619228e-05,
5.58810752e-05, 4.91898136e-05
],
[
5.09982638e-05, 4.77430549e-05, 4.97323490e-05,
5.43438946e-05, 5.26258664e-05
]]]
assert (np.isclose(data, expected_data).all())
def test_variable_esgf_query(self):
assert(os.environ.get('OPENID_ESGF'))
assert(os.environ.get('PASSWORD_ESGF'))
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=self.url)
# Ensure authentication:
res = pydap.net.follow_redirect(self.test_url, session=session)
assert(res.status_code == 200)
dataset = open_url(self.url, session=session, output_grid=False)
data = dataset['pr'][0, 200:205, 100:105]
expected_data = [[[5.23546005e-05, 5.48864300e-05,
5.23546005e-05, 6.23914966e-05,
6.26627589e-05],
[5.45247385e-05, 5.67853021e-05,
5.90458621e-05, 6.51041701e-05,
6.23914966e-05],
[5.57906533e-05, 5.84129048e-05,
6.37478297e-05, 5.99500854e-05,
5.85033267e-05],
[5.44343166e-05, 5.45247385e-05,
5.60619228e-05, 5.58810752e-05,
4.91898136e-05],
[5.09982638e-05, 4.77430549e-05,
4.97323490e-05, 5.43438946e-05,
5.26258664e-05]]]
assert(np.isclose(data, expected_data).all())
def test_dimension_esgf_query(self):
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=self.url)
dataset = open_url(self.url, session=session)
data = dataset['time'][:10]
expected_data = np.array([
16832.5, 16833.5, 16834.5, 16835.5, 16836.5, 16837.5, 16838.5,
16839.5, 16840.5, 16841.5
])
assert (np.isclose(data, expected_data).all())
def test_variable_esgf_query():
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=url)
# Ensure authentication:
res = pydap.net.follow_redirect(test_url, session=session)
assert(res.status_code == 200)
dataset = open_url(url, session=session, output_grid=False)
data = dataset['orog'][103:105, 100:102]
expected_data = [[271.36645508, 166.85339355],
[304.22286987, 178.85267639]]
assert(np.isclose(data, expected_data).all())
def test_dimension_esgf_query():
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=url)
# Ensure authentication:
res = pydap.net.follow_redirect(test_url, session=session)
assert(res.status_code == 200)
dataset = open_url(url, session=session)
data = dataset['lon'][:5]
expected_data = np.array([0., 1.25, 2.5, 3.75, 5.])
assert(np.isclose(data, expected_data).all())
def test_variable_esgf_query():
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=url)
# Ensure authentication:
res = pydap.net.follow_redirect(test_url, session=session)
assert (res.status_code == 200)
dataset = open_url(url, session=session, output_grid=False)
data = dataset['orog'][103:105, 100:102]
expected_data = [[271.36645508, 166.85339355],
[304.22286987, 178.85267639]]
assert (np.isclose(data, expected_data).all())
def test_dimension_esgf_query():
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=url)
# Ensure authentication:
res = pydap.net.follow_redirect(test_url, session=session)
assert(res.status_code == 200)
dataset = open_url(url, session=session)
data = dataset['lon'][:5]
expected_data = np.array([0.0, 2.8125, 5.625, 8.4375, 11.25])
assert(np.isclose(data, expected_data).all())
def test_variable_esgf_query():
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=url)
# Ensure authentication:
res = pydap.net.follow_redirect(test_url, session=session)
assert(res.status_code == 200)
dataset = open_url(url, session=session, output_grid=False)
data = dataset['orog'][50:55, 50:55]
expected_data = [[197.70425, 16.319595, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[677.014, 628.29675, 551.06, 455.5758, 343.7354],
[1268.3304, 1287.9553, 1161.0402, 978.3153, 809.143]]
assert(np.isclose(data, expected_data).all())
def test_dimension_esgf_query(self):
assert(os.environ.get('OPENID_ESGF'))
assert(os.environ.get('PASSWORD_ESGF'))
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=self.url)
# Ensure authentication:
res = pydap.net.follow_redirect(self.test_url, session=session)
assert(res.status_code == 200)
dataset = open_url(self.url, session=session)
data = dataset['time'][:10]
expected_data = np.array([16832.5, 16833.5, 16834.5, 16835.5, 16836.5,
16837.5, 16838.5, 16839.5, 16840.5, 16841.5])
assert(np.isclose(data, expected_data).all())
def _get_session(self):
if self.auth is None:
session = None
else:
if self.auth == "esgf":
from pydap.cas.esgf import setup_session
elif self.auth == "urs":
from pydap.cas.urs import setup_session
else:
raise ValueError(
"Authentication method should either be None, 'esgf' or 'urs', "
f"got '{self.auth}' instead."
)
username = os.getenv('DAP_USER', None)
password = os.getenv('DAP_PASSWORD', None)
session = setup_session(username, password, check_url=self.urlpath)
return session
def test_dimension_esgf_query(self):
assert (os.environ.get('OPENID_ESGF'))
assert (os.environ.get('PASSWORD_ESGF'))
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=self.url)
# Ensure authentication:
res = pydap.net.follow_redirect(self.test_url, session=session)
assert (res.status_code == 200)
dataset = open_url(self.url, session=session)
data = dataset['time'][:10]
expected_data = np.array([
16832.5, 16833.5, 16834.5, 16835.5, 16836.5, 16837.5, 16838.5,
16839.5, 16840.5, 16841.5
])
assert (np.isclose(data, expected_data).all())
def test_basic_esgf_auth():
"""
Set up PyDAP to use the ESGF request() function.
The intent here is to ensure that pydap.net is able to
open and url if and only if requests is able to
open the same url.
"""
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=url)
res = requests.get(test_url, cookies=session.cookies)
assert (res.status_code == 200)
res.close()
res = pydap.net.follow_redirect(test_url, session=session)
assert (res.status_code == 200)
def test_basic_esgf_auth():
"""
Set up PyDAP to use the ESGF request() function.
The intent here is to ensure that pydap.net is able to
open and url if and only if requests is able to
open the same url.
"""
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=url)
res = requests.get(test_url, cookies=session.cookies)
assert(res.status_code == 200)
res.close()
res = pydap.net.follow_redirect(test_url, session=session)
assert(res.status_code == 200)
def DownloadPoints(variable,
latitudes_s,
longitudes_s,
url,
outfile,
session=None):
""" Download time-series for a list of lats/longs from a single opendap dataset (ie CORDEX netcdf).
The output nc file has primary dimensions location and time.
Inputs:
-------
variable - string. variable to download, eg 'tas' or 'pr'
latitudes_s, longitudes_s are python lists
url - opendap url
outfile - string. where to write the file.
session - Note or pydap.cas.esgf session. None for a new session.
Might save a little time if you reuse sessions rather than creating new
ones for each download? Might be more trouble than it is worth, seems to
get time-outs with re-using sesions. Best to just leave as None. But if
you want to try:
session = setup_session(config.openid, config.password, verify=True, check_url=url)
"""
if session == None:
session = setup_session(config.openid,
config.password,
verify=True,
check_url=url)
d = open_url(url, session=session)
# lat/long to grid mapping
Message('download from %s' % url, 2)
####
Message(' ... reading header and extracting pixel indices...',
2,
newline=False)
lat = d['lat'].array.data[:, :]
lon = d['lon'].array.data[:, :]
for i, l in enumerate(longitudes_s):
if l > 180:
longitudes_s[i] = l - 360
lon_shift = lon > 180
lon[lon_shift] = lon[lon_shift] - 360
ns = len(longitudes_s)
rlat_idx = [None] * ns
rlon_idx = [None] * ns
lat_of_pixel = np.zeros(ns, dtype=lat.dtype)
lon_of_pixel = np.zeros(ns, dtype=lat.dtype)
Message('done.', 2)
####
Message(' ... reading and processing header for %d locations...' % ns, 2,
False)
for i in range(ns):
longitude = longitudes_s[i]
latitude = latitudes_s[i]
dist = ((abs(lon - longitude)**2) + (abs(lat - latitude)**2))
mask = dist == np.amin(dist)
lat_of_pixel[i] = lat[mask]
lon_of_pixel[i] = lon[mask]
rlat_idx[i] = np.asscalar(mask.nonzero()[0])
rlon_idx[i] = np.asscalar(mask.nonzero()[1])
Message('done.', 2)
####
# open a new netCDF file for writing, and download data and write it in directly!
Message(' ... downloading and writing to %s...' % outfile, 2)
with Dataset(outfile, 'w', format='NETCDF4') as ncfile:
for name in d.attributes['NC_GLOBAL']:
attr_value = d.attributes['NC_GLOBAL'][name]
if name[0] != '_' and isinstance(attr_value, str):
ncfile.setncattr(name, attr_value)
# create the x and y dimensions.
ncfile.createDimension('location', ns)
ncfile.createDimension('time', None)
times = ncfile.createVariable('time', d['time'].dtype.name, ('time', ))
latitudes = ncfile.createVariable('lat', lat_of_pixel.dtype.name,
('location', ))
longitudes = ncfile.createVariable('lon', lon_of_pixel.dtype.name,
('location', ))
M = ncfile.createVariable(variable, d[variable].array.dtype.name, (
'time',
'location',
))
latitudes[:] = lat_of_pixel
longitudes[:] = lon_of_pixel
times[:] = d['time'].data[:]
for i in range(ns):
Message(' ... station %d of %d' % (i, ns), 2)
M[:, i] = d[variable].array.data[:, rlat_idx[i], rlon_idx[i]]
for cvar in (
'time',
variable,
):
for name in d[cvar].attributes:
attr_value = d[cvar].attributes[name]
if name[0] != '_' and isinstance(attr_value, str):
ncfile.variables[cvar].setncattr(name, attr_value)
Message('done.', 2)
def DownloadSlice(variable,
url,
outfile,
rlat_idx=None,
rlon_idx=None,
time_idx=None,
session=None):
""" Download a slice for a single opendap dataset (ie CORDEX netcdf).
Inputs:
-------
variable - string. variable to download, eg 'tas' or 'pr'
url - opendap url.
outfile - string. where to write the file.
rlat_idx, rlon_idx, time_idx - slices for downloading a subset of the grids. eg:
rlat_idx=slice(250,391)
rlon_idx=slice(198,286)
None for the entire axis (all that is supported for time??)
session - Note or pydap.cas.esgf session. None for a new session.
Might save a little time if you reuse sessions rather than creating new
ones for each download? Might be more trouble than it is worth, seems to
get time-outs with re-using sesions. Best to just leave as None. But if
you want to try:
session = setup_session(config.openid, config.password, verify=True, check_url=url)
"""
if session == None:
session = setup_session(config.openid,
config.password,
verify=True,
check_url=config.check_url)
Message('download from %s' % url, 2)
####
# defaults for all! presume this actually works...
if rlat_idx == None:
rlat_idx = slice(0, None)
if rlon_idx == None:
rlon_idx = slice(0, None)
if time_idx == None:
time_idx = slice(0, None)
####
d = open_url(url, session=session)
template = d['lat'].array.data[rlat_idx, rlon_idx]
# open a new netCDF file for writing, and download data and write it in directly!
Message(' ... downloading and writing to %s...' % outfile, 2)
#with Dataset(outfile,'w', format='NETCDF4') as ncfile:
ncfile = Dataset(outfile, 'w', format='NETCDF4')
for name in d.attributes['NC_GLOBAL']:
attr_value = d.attributes['NC_GLOBAL'][name]
if name[0] != '_' and isinstance(attr_value, str):
ncfile.setncattr(name, attr_value)
# create the x and y dimensions.
ncfile.createDimension('rlon', template.shape[1])
ncfile.createDimension('rlat', template.shape[0])
ncfile.createDimension('time', None)
times = ncfile.createVariable('time', d['time'].dtype.name, ('time', ))
rlats = ncfile.createVariable('rlat', d['rlat'].dtype.name, ('rlat', ))
rlons = ncfile.createVariable('rlon', d['rlat'].dtype.name, ('rlon', ))
latitudes = ncfile.createVariable('lat', d['lat'].array.dtype.name, (
'rlat',
'rlon',
))
longitudes = ncfile.createVariable('lon', d['lat'].array.dtype.name, (
'rlat',
'rlon',
))
M = ncfile.createVariable(variable, d[variable].array.dtype.name, (
'time',
'rlat',
'rlon',
))
rlats[:] = d['rlat'].data[rlat_idx]
rlons[:] = d['rlon'].data[rlon_idx]
latitudes[:, :] = d['lat'].array.data[rlat_idx, rlon_idx]
longitudes[:, :] = d['lon'].array.data[rlat_idx, rlon_idx]
times[:] = d['time'].data[:]
M[:, :, :] = d[variable].array.data[time_idx, rlat_idx, rlon_idx]
for cvar in ('time', variable, 'rlat', 'rlon', 'lat', 'lon'):
for name in d[cvar].attributes:
attr_value = d[cvar].attributes[name]
if name[0] != '_' and isinstance(attr_value, str):
ncfile.variables[cvar].setncattr(name, attr_value)
ncfile.close()
Message('done.', 2)
def _get_session(self):
from pydap.cas.esgf import setup_session
username = os.getenv('DAP_USER', None)
password = os.getenv('DAP_PASSWORD', None)
return setup_session(username, password, check_url=self.urlpath)
"""
Example of using CORDEX_DownloadFunctions to download a sub-grid of CORDEX data
"""
import os
#from pydap.client import open_url
from pydap.cas.esgf import setup_session
from CORDEX_DownloadFunctions import JSONParser, ConcatenatedCatalogSlice
import config
# this is just to check that things are working...
session = setup_session(config.openid,
config.password,
verify=True,
check_url=config.check_url)
# ~Norden in EU44
#rlat_idx=slice(61,101)
#rlon_idx=slice(48,74)
# ~Norden in EU11
rlat_idx = slice(250, 391)
rlon_idx = slice(198, 286)
ModelSelectionFilename = os.path.join('CORDEX_Example', 'monthly_test.json')
Outdir = os.path.join('CORDEX_Example', 'Data')
p = JSONParser(ModelSelectionFilename)
ConcatenatedCatalogSlice(p.Catalog,
