def windspeed(self):
varu="u-component_of_wind_surface"
varv="v-component_of_wind_surface"
uVel=[]
vVel=[]
for month_year_str in self.__years_months:
month,year=month_year_str.split("-")
days=monthrange(int(year),int(month))[1]
base_url=self.__urlfunc("wind",month,year)
if(isinstance(self.__lon1,int)):
url=base_url+varu+self.__varstr(days)+","+varv+self.__varstr(days)
print(url)
request=open_dods(url)
#print(month,year,time.time()-t)
#u_vel=request[varu][varu]__calc_maxt_idx__calc_maxt_idx__calc_maxt_idx__calc_maxt_idx
#v_vel=request[varv][varv]
u_vel=request[varu]
v_vel=request[varv]
uVel.append(u_vel)
vVel.append(v_vel)
else:
url1=base_url+varu+self.__varstr[0](days)+","+varv+self.__varstr[0](days)
url2=base_url+varu+self.__varstr[1](days)+","+varv+self.__varstr[1](days)
request1=open_dods(url1)
request2=open_dods(url2)
u_vel1=request1[varu]
v_vel1=request1[varv]
u_vel2=request2[varu]
v_vlatsrangeel2=request2[varv]
uVel.append((u_vel1,u_vel2))
vVel.append((v_vel1,v_vel2))
return uVel,vVel
def waveheight(self):
var_height="Significant_height_of_combined_wind_waves_and_swell_surface"
waveheights=[]
for month_year_str in self.__years_months:
month,year=month_year_str.split("-")
days=monthrange(int(year),int(month))[1]
base_url=self.__urlfunc("hs",month,year)
if(isinstance(self.__lon1,int)):
url=base_url+var_height+self.__varstr(days)
#print(url)
print(month_year_str,url)
request=open_dods(url)
wave_height=request[var_height]
waveheights.append(wave_height)
else:
url1=base_url+var_height+self.__varstr[0](days)
url2=base_url+var_height+self.__varstr[1](days)
#print(url1,url2)
request1=open_dods(url1)
request2=open_dods(url2)
wave_height1=request1[var_height]
wave_height2=request2[var_height]
waveheights.append((wave_height1,wave_height2))
return waveheights
def __init__(self,):
self.__base_url="https://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_93.0/uv3z.dods?"
self.__start_date="2000-01-01 00:00:00"
self.__mindepth=self.__maxdepth=0
self.__minlat=-80
self.__minlon=0
self.__maxlon=359.92
url=self.__base_url+"time"
laturl=self.__base_url+"lat"
print(url)
self._timesmap=np.asarray(open_dods(url)["time"])
self._latsmap=np.around(np.asarray(open_dods(laturl)["lat"]),2)
def test_timeout(sequence_type_data):
"""Test that timeout works properly"""
TestDataset = DatasetType('Test')
TestDataset['sequence'] = sequence_type_data
TestDataset['byte'] = BaseType('byte', 0)
application = BaseHandler(TestDataset)
# Explictly add latency on the devel server
# to guarantee that it timeouts
def wrap_mocker(func):
def mock_add_latency(*args, **kwargs):
time.sleep(1e-1)
return func(*args, **kwargs)
return mock_add_latency
application = wrap_mocker(application)
with LocalTestServer(application) as server:
url = ("http://0.0.0.0:%s/" % server.port)
# test open_url
assert open_url(url) == TestDataset
with pytest.raises(HTTPError) as e:
open_url(url, timeout=1e-5)
assert 'Timeout' in str(e)
# test open_dods
with pytest.raises(HTTPError):
open_dods(url + '.dods?sequence', timeout=1e-5)
assert 'Timeout' in str(e)
# test sequenceproxy
dataset = open_url(url)
seq = dataset['sequence']
assert isinstance(seq.data, SequenceProxy)
# Change the timeout of the sequence proxy:
seq.data.timeout = 1e-5
with pytest.raises(HTTPError) as e:
next(seq.iterdata())
assert 'Timeout' in str(e)
# test baseproxy:
dat = dataset['byte']
assert isinstance(dat.data, BaseProxy)
# Change the timeout of the baseprox proxy:
dat.data.timeout = 1e-5
with pytest.raises(HTTPError) as e:
dat[:]
assert 'Timeout' in str(e)
def test_timeout(sequence_type_data):
"""Test that timeout works properly"""
TestDataset = DatasetType('Test')
TestDataset['sequence'] = sequence_type_data
TestDataset['byte'] = BaseType('byte', 0)
application = BaseHandler(TestDataset)
# Explictly add latency on the devel server
# to guarantee that it timeouts
def wrap_mocker(func):
def mock_add_latency(*args, **kwargs):
time.sleep(1e-1)
return func(*args, **kwargs)
return mock_add_latency
application = wrap_mocker(application)
with LocalTestServer(application) as server:
url = ("http://0.0.0.0:%s/" % server.port)
# test open_url
assert open_url(url) == TestDataset
with pytest.raises(HTTPError) as e:
open_url(url, timeout=1e-5)
assert 'Timeout' in str(e)
# test open_dods
with pytest.raises(HTTPError):
open_dods(url + '.dods?sequence', timeout=1e-5)
assert 'Timeout' in str(e)
# test sequenceproxy
dataset = open_url(url)
seq = dataset['sequence']
assert isinstance(seq.data, SequenceProxy)
# Change the timeout of the sequence proxy:
seq.data.timeout = 1e-5
with pytest.raises(HTTPError) as e:
next(seq.iterdata())
assert 'Timeout' in str(e)
# test baseproxy:
dat = dataset['byte']
assert isinstance(dat.data, BaseProxy)
# Change the timeout of the baseprox proxy:
dat.data.timeout = 1e-5
with pytest.raises(HTTPError) as e:
dat[:]
assert 'Timeout' in str(e)
def testOpenDap(self):
print 'testOpenDap NASA EARTHDATA'
# Get authentication from the .netrc file
netrc_file = '/home/jli/.netrc'
if os.path.isfile(netrc_file):
logins = netrc.netrc()
accounts = logins.hosts
for host, info in accounts.iteritems():
self.login, self.account, self.password = info
#dataset_url = 'https://goldsmr5.gesdisc.eosdis.nasa.gov:443/opendap/MERRA2/M2I3NPASM.5.12.4/1986/12/MERRA2_100.inst3_3d_asm_Np.19861201.nc4'
dataset_url = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/dods/M2T1NXSLV.dods?t2m[1:2][1:5][7:14]'
dataset_url = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/dods/M2T1NXSLV.dods?t2m'
session = setup_session(self.login,
self.password,
check_url=dataset_url)
#dataset = open_url(dataset_url,session=session)
#dataset_url = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/dods/M2T1NXSLV.dods?t2m[1:2][1:5][7:14]'
dataset = open_dods(dataset_url, session=session)
#store = xarray.backends.PydapDataStore(dataset)
#ds = xarray.open_dataset(store)
#print ds.keys()
print type(dataset)
print dataset.keys()
def getData(self):
todayString = self.gmTime.strftime('%Y%m%d')
global baseurl
oururl = baseurl + todayString + '/nww3' + \
todayString + '_%02dz' % self.tm_hour
dodsUrl = oururl + ".dods?" + \
"{2}.{2}[0:1:60][{0}:1:{0}][{1}:1:{1}]".format(
self.lattitudeIndex,
self.longitudeIndex,
self.variable)
logging.debug('DODS:' + dodsUrl)
try:
data = open_dods(dodsUrl)
except ServerError:
logging.debug('URL DOES NOTEXIST')
self.gmTime -= timedelta(hours=6)
self.tm_hour = chooseNearestHour(self.gmTime)
return self.getData()
#logging.debug(data[self.variable][:])
#logging.debug(type(data))
#logging.debug(type(data[self.variable]))
for key,value in enumerate(data[self.variable]):
if value>9.999e+20:
#logging.debug('No data;'+str(key)+':'+str(value))
data[self.variable].data[key]=NODATA
return data[self.variable][:]
def getData(self):
todayString = self.gmTime.strftime('%Y%m%d')
global baseurl
oururl = baseurl + todayString + '/nww3' + \
todayString + '_%02dz' % self.tm_hour
dodsUrl = oururl + ".dods?" + \
"{2}.{2}[0:1:60][{0}:1:{0}][{1}:1:{1}]".format(
self.lattitudeIndex,
self.longitudeIndex,
self.variable)
logging.debug('DODS:' + dodsUrl)
try:
data = open_dods(dodsUrl)
except ServerError:
logging.debug('URL DOES NOTEXIST')
self.gmTime -= timedelta(hours=6)
self.tm_hour = chooseNearestHour(self.gmTime)
return self.getData()
#logging.debug(data[self.variable][:])
#logging.debug(type(data))
#logging.debug(type(data[self.variable]))
for key, value in enumerate(data[self.variable]):
if value > 9.999e+20:
#logging.debug('No data;'+str(key)+':'+str(value))
data[self.variable].data[key] = NODATA
return data[self.variable][:]
def explore():
d = open_url('http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc')
##v = d["SST"]
##
##v["lon"]
##
##v["SST"].shape
##
##s = v["SST"]
##
##print
##(1, 16000, 36000)
##a = d.SST[0,12000:13000,18000:19000] # time:0, lat: 40-50N, lon 0-10E
sst = d['SST']
##grid = sst[0:1:0,0:1:10,0:1:3]
##grid = sst[0:1:0,100:1:110,0:1:3]
# generated http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.dods?SST.SST[0:1:0][100:110:0][0:3:0]
# which is not correct, it should be:
# http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.ascii?SST[0:1:0][0:1:10][0:1:3]
##grid = sst.SST[d.time==0, (d.lat >= 100) & (d.lat <= 110), (d.lon >= 0) & (d.lon <= 3)]
from pydap.client import open_dods
d2 = open_dods('http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.dods?SST[0:1:0][0:1:10][0:1:3]')
##http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.ascii?SST[0:1:0][1000:1:1010][1000:1:1003]
#d = open_url('http://coastwatch.pfeg.noaa.gov/erddap/griddap/erdTAssh1day')
##start = end = datetime(2010, 5, 19, 12, 0, 0)
##ssh = d.ssh.ssh[
## (d.time >= coards.format(start, d.time.units)) & (d.time <= coards.format(end, d.time.units)),
## (d.altitude == 0),
## (d.latitude >= 17) & (d.latitude <= 32),
## (d.longitude >= 260) & (d.longitude <= 281)]
#http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.ascii?SST[0:1:0][100:1:110][0:1:3]
suffix = "2013/147/20130527-JPL_OUROCEAN-L4UHfnd-GLOB-v01-fv01_0-G1SST.nc.bz2"
d = open_url("http://podaac-opendap.jpl.nasa.gov/opendap/allData/ghrsst/data/L4/GLOB/JPL_OUROCEAN/G1SST/" + suffix)
##g = d.analysed_sst[0, 12000:12010, 18000:18010]
g.lat.data
g.lon.data
t = g.analysed_sst.data
def test_open_dods_with_attributes(self):
"""Open the dods response together with the das response."""
dataset = open_dods('.dods', metadata=True, application=self.app)
self.assertEqual(dataset.NC_GLOBAL, {})
self.assertEqual(dataset.DODS_EXTRA, {})
self.assertEqual(dataset.description, "A simple sequence for testing.")
self.assertEqual(dataset.cast.lon.axis, 'X')
self.assertEqual(dataset.cast.lat.axis, 'Y')
self.assertEqual(dataset.cast.depth.axis, 'Z')
self.assertEqual(dataset.cast.time.axis, 'T')
self.assertEqual(dataset.cast.time.units, "days since 1970-01-01")
def test_open_dods_with_attributes(sequence_app):
"""Open the dods response together with the das response."""
dataset = open_dods('.dods', metadata=True, application=sequence_app)
assert (dataset.NC_GLOBAL == {})
assert (dataset.DODS_EXTRA == {})
assert (
dataset.description == "A simple sequence for testing.")
assert (dataset.cast.lon.axis == 'X')
assert (dataset.cast.lat.axis == 'Y')
assert (dataset.cast.depth.axis == 'Z')
assert (dataset.cast.time.axis == 'T')
assert (
dataset.cast.time.units == "days since 1970-01-01")
def wave_speeds(self,latrange,lonrange,timerange):
u_name="water_u"
v_name="water_v"
minlat,maxlat=self.__map_lats(latrange)
minlon,maxlon=self.__map_lons(lonrange)
startidx,endidx=self._map_times(timerange)
time_string="[{}:1:{}]".format(startidx,endidx)
depth_string="[{}:1:{}]".format(self.__mindepth,self.__maxdepth)
lat_string="[{}:1:{}]".format(minlat,maxlat)
if(isinstance(minlon,int)):
lon_string="[{}:1:{}]".format(minlon,maxlon)
else:
lon_string=("[{}:1:{}]".format(minlon[0],minlon[1]),"[{}:1:{}]".format(maxlon[0],maxlon[1]))
if(isinstance(minlon,int)):
u_string=u_name+time_string+depth_string+lat_string+lon_string
v_string=v_name+time_string+depth_string+lat_string+lon_string
url=self.__base_url+u_string+","+v_string
req=open_dods(url,timeout=5000)
u=req[u_name]
v=req[v_name]
return u,v
else:
u_string1=u_name+time_string+depth_string+lat_string+lon_string[0]
v_string1=v_name+time_string+depth_string+lat_string+lon_string[0]
u_string2=u_name+time_string+depth_string+lat_string+lon_string[1]
v_string2=v_name+time_string+depth_string+lat_string+lon_string[1]
url1=self.__base_url+u_string1+","+v_string1
url2=self.__base_url+u_string2+","+v_string2
req1=open_dods(url1,timeout=5000)
req2=open_dods(url2,timeout=5000)
u1=req1[u_name]
v1=req1[v_name]
u2=req2[u_name]
v2=req2[v_name]
u=(u1,u2)
v=(v1,v2)
return u,v
def test_open_dods_with_attributes(self):
"""Open the dods response together with the das response."""
dataset = open_dods('.dods', metadata=True, application=self.app)
self.assertEqual(dataset.NC_GLOBAL, {})
self.assertEqual(dataset.DODS_EXTRA, {})
self.assertEqual(
dataset.description, "A simple sequence for testing.")
self.assertEqual(dataset.cast.lon.axis, 'X')
self.assertEqual(dataset.cast.lat.axis, 'Y')
self.assertEqual(dataset.cast.depth.axis, 'Z')
self.assertEqual(dataset.cast.time.axis, 'T')
self.assertEqual(
dataset.cast.time.units, "days since 1970-01-01")
def compute(self):
url = self.getInputFromPort("url")
dataSlice = self.getInputFromPort("dataSlice")
myDataSet = open_dods(str(url) + '.dods?' + str(dataSlice))
result = myDataSet.data
#TODO: should return list of arrays but this will require a new type
# to be added to eo4vistrails; for instance NDArrayList
my_first_data_item_key = myDataSet.keys()[0]
dict_of_arrays = myDataSet.get(my_first_data_item_key)
for output_port, known_field_name in (('time','time'),
('latitude','latitude'),
('longitude','longitude'),
('data',my_first_data_item_key)):
outArray = NDArray()
outArray.set_array(dict_of_arrays.get(known_field_name).data)
self.setResult(output_port, outArray)
def test_open_dods(self):
"""Open the dods response from a server.
Note that here we cannot simply compare ``dataset.data`` with the
original data ``SimpleSequence.data``, because the original data
contains int16 values which are transmitted as int32 in the DAP spec.
"""
dataset = open_dods('.dods', application=self.app)
self.assertEqual(list(dataset.data), [[
('1', 100, -10, 0, -1, 21, 35, 0),
('2', 200, 10, 500, 1, 15, 35, 100),
]])
# attributes should be empty
self.assertEqual(dataset.attributes, {})
def test_open_dods_16bits(sequence_app):
"""Open the dods response from a server.
Note that here we cannot simply compare ``dataset.data`` with the
original data ``SimpleSequence.data``, because the original data
contains int16 values which are transmitted as int32 in the DAP spec.
"""
dataset = open_dods('.dods', application=sequence_app)
assert (
list(dataset.data) == [[
('1', 100, -10, 0, -1, 21, 35, 0),
('2', 200, 10, 500, 1, 15, 35, 100),
]])
# attributes should be empty
assert dataset.attributes == {}
def subset_dataset(self, url, metadata=False):
""" Return a subset dataset as
Parameters
----------
url : [type]
base URL for a dataset that can be opened with pydap.open_url
Returns
-------
DatasetType : pydap DatasetType
"""
dods_url = self.create_dods_url(url)
logger.debug(f"Downloading subset from {dods_url}")
ds = open_dods(dods_url, session=self.session, metadata=metadata)
return ds
def wave_speeds2(self,latrange,lonrange,timerange,foldername):
df=pd.DataFrame(columns=["date_time","lat","lon","water_u","water_v"])
u_name="water_u"
v_name="water_v"
minlat,maxlat=self.__map_lats(latrange)
minlon,maxlon=self.__map_lons(lonrange)
startidx,endidx=self._map_times(timerange)
print(startidx,endidx)
start_date=datetime.strptime("2000-01-01 00:00:00","%Y-%m-%d %H:%M:%S")
depth_string="[{}:1:{}]".format(self.__mindepth,self.__maxdepth)
lat_string="[{}:1:{}]".format(minlat,maxlat)
if(isinstance(minlon,int)):
lon_string="[{}:1:{}]".format(minlon,maxlon)
else:
lon_string=("[{}:1:{}]".format(minlon[0],minlon[1]),"[{}:1:{}]".format(maxlon[0],maxlon[1]))
if(isinstance(minlon,int)):
lon_strings=[lon_string]
else:
lon_strings=[lon_string[0],lon_string[1]]
for lonid,lon_string in enumerate(lon_strings):
for idx in range(startidx,endidx+1):
time_string="[{}:1:{}]".format(idx,idx)
if(not os.path.exists(foldername+"/"+str(idx)+"_"+str(lonid)+".npy")):
u_string=u_name+time_string+depth_string+lat_string+lon_string
v_string=v_name+time_string+depth_string+lat_string+lon_string
url=self.__base_url+u_string+","+v_string
req=open_dods(url,timeout=50000)
u=req[u_name]
v=req[v_name]
times=int(np.asarray(u["time"])[0])
lats=u["lat"]
lons=u["lon"]
uv=[u["water_u"][0],v["water_v"][0]]
if(idx==startidx):
if(lonid==0):
np.save(foldername+"/"+"lats",np.asarray(lats))
np.save(foldername+"/"+"lons_{}".format(lonid),np.asarray(lons))
np.save(foldername+"/"+str(idx)+"_"+str(lonid),np.asarray(uv))
return True
def get_sequential(file, time, var_config, lat_idx, lon_idx, verbose=False):
var_list = []
nlev_dict = {}
for var, config in var_config.items():
if config["type"] == "surface":
var_list.append(
FORMAT_STR.format(var=var, lat=lat_idx, lon=lon_idx))
nlev_dict[var] = 1
else:
lev_idx = tuple(config["levels"])
var_list.append(
FORMAT_STR_PL.format(var=var,
lev=lev_idx,
lat=lat_idx,
lon=lon_idx))
nlev_dict[var] = lev_idx[1] - lev_idx[0] + 1
ncoord = (lat_idx[1] - lat_idx[0] + 1) * (lon_idx[1] - lon_idx[0] + 1)
request = URL.format(file, time) + ",".join(var_list)
if verbose:
print(request)
try:
dataset = open_dods(request)
except:
raise
var_data = [
pd.DataFrame(
var.data.reshape(nlev_dict[var.name], ncoord).T,
columns=[
"{}{}".format(var.name, n) for n in range(nlev_dict[var.name])
],
) for var in dataset
]
return pd.concat(var_data, axis=1)
def get_file(request, param, var_conf, time, lat, lon, verbose=False):
ntime = len(time)
ncoord = len(lat) * len(lon)
var_list = [
((FORMAT_STR if vartype == "surface" else FORMAT_STR_PL).format(
var=var, **param)) for var, vartype in var_conf.items()
]
if verbose:
print(request + ",".join(var_list))
try:
dataset = open_dods(request + ",".join(var_list))
except:
raise OpenFileError("file '{}' not available".format(request[:-1]))
var_data = [
var.data.reshape((ntime, -1, ncoord)) for var in dataset.values()
]
var_names = [
"{}{}".format(var, n) for idx, var in enumerate(dataset)
for n in range(var_data[idx].shape[1])
]
index = pd.MultiIndex.from_product((lat, lon), names=["lat", "lon"])
columns = pd.MultiIndex.from_product((time, var_names),
names=["time", "var"])
return pd.DataFrame(
(np.concatenate(var_data, axis=1).transpose(2, 0, 1).reshape(
ncoord, -1)),
index=index,
columns=columns,
)
#!/usr/bin/env python
from pydap.client import open_dods
import numpy as np
import argparse as ap
parser = ap.ArgumentParser(description='Use pydap client to access HDF5 files via Hyrax ')
parser.add_argument('url', nargs='?',default='https://eosdap.hdfgroup.org:8080/opendap/data/hdf5/grid_1_2d.h5.dods?temperature[0:1:1][0:1:1]',help='The URL to access the HDF5 files via Hyrax')
#Change the URL to your own server's address
args=parser.parse_args()
input_url =''.join(args.url)
#Obtain the data via pydap
dataset=open_dods(input_url)
data=[]
data=dataset.temperature[:]
#Generate the output file name from URL
last_slash_index=input_url.rfind('/')
h5_suffix_index=input_url.find(".h5",last_slash_index)
output_string=input_url[last_slash_index+1:h5_suffix_index]
#Save the data with numpy savetxt
np.savetxt(output_string+'_data.txt',data)
def save_dataset(hour,
date,
var_config,
time_tuple,
lat_tuple,
lon_tuple,
fname,
verbose=False):
"""Download the datasets for a specific date and hour"""
date_str = date.strftime("%Y%m%d")
month_str = date.strftime("%Y%m")
file = DIR.format(month_str, date_str, hour)
time_list = list(range1(time_tuple[0], time_tuple[1], 3))
# Get the lat and lon grids from the first dataset present in the server
for time in time_list:
request = URL.format(file, time)
if verbose:
print(request)
try:
coord = open_dods(request + "lat,lon")
except:
continue
else:
break
try:
lat, lon = coord["lat"][:].data, coord["lon"][:].data
except:
# UnboundLocalError: local variable 'coord' referenced before assignment
# none of the 180/3 + 1 datasets where present in the server
raise
# Transform longitudes from range 0..360 to -180..180
lon = np.where(lon > 180, lon - 360, lon)
# Transform into python lists to use the index() method
# Actually it would be better to use np.where()
lat_list, lon_list = lat.tolist(), lon.tolist()
try:
lat_idx = (lat_list.index(lat_tuple[1]), lat_list.index(lat_tuple[0]))
except:
raise ValueError("Latitude not in the grid", lat_tuple)
lat = lat[range1(*lat_idx)].tolist()
if lon_tuple[0] < 0 and lon_tuple[1] > 0:
try:
lon_idx_w = (lon_list.index(lon_tuple[0]), len(lon_list) - 1)
lon_idx_e = (0, lon_list.index(lon_tuple[1]))
except:
raise ValueError("Longitude not in the grid", lon_tuple)
lon = np.concatenate(
(lon[range1(*lon_idx_w)], lon[range1(*lon_idx_e)])).tolist()
try:
data_list = [
get_general(
file,
time,
var_config,
lat_idx,
lon_idx_w,
lon_idx_e,
verbose=verbose,
) for time in time_list
]
except:
raise
else:
try:
lon_idx = (lon_list.index(lon_tuple[0]),
lon_list.index(lon_tuple[1]))
except:
raise ValueError("Longitude not in the grid", lon_tuple)
lon = lon[range1(*lon_idx)].tolist()
try:
data_list = [
get_sequential(file,
time,
var_config,
lat_idx,
lon_idx,
verbose=verbose) for time in time_list
]
except:
raise
data = pd.concat(data_list, axis=1, keys=time_list, names=["time", "var"])
data.index = pd.MultiIndex.from_product((lat, lon), names=["lat", "lon"])
data.sort_index(inplace=True)
data.to_csv(fname, sep=" ", float_format="%.3f")
def get_general(file,
time,
var_config,
lat_idx,
lon_idx_w,
lon_idx_e,
verbose=False):
request = URL.format(file, time)
var_w_list, var_e_list = [], []
nlev_dict = {}
for var, config in var_config.items():
if config["type"] == "surface":
var_w_list.append(
FORMAT_STR.format(var=var, lat=lat_idx, lon=lon_idx_w))
var_e_list.append(
FORMAT_STR.format(var=var, lat=lat_idx, lon=lon_idx_e))
nlev_dict[var] = 1
else:
lev_idx = tuple(config["levels"])
var_w_list.append(
FORMAT_STR_PL.format(var=var,
lev=lev_idx,
lat=lat_idx,
lon=lon_idx_w))
var_e_list.append(
FORMAT_STR_PL.format(var=var,
lev=lev_idx,
lat=lat_idx,
lon=lon_idx_e))
nlev_dict[var] = lev_idx[1] - lev_idx[0] + 1
ncoord = (lat_idx[1] - lat_idx[0] + 1) * (
(lon_idx_w[1] - lon_idx_w[0] + 1) + (lon_idx_e[1] - lon_idx_e[0] + 1))
request_w = request + ",".join(var_w_list)
request_e = request + ",".join(var_e_list)
if verbose:
print(request_w)
print(request_e)
try:
dataset_w = open_dods(request_w)
dataset_e = open_dods(request_e)
except:
raise
# 1. Concatenate both 'west' and 'east' data along 'lon' axis (last axis)
# 2. Reshape 4D-array (1, nlev, nlat, nlon) to 2D-array (nlev, nlat*nlon)
# 3. Transpose and store in DataFrame
# This is ugly :(, partly because the long function names
var_data = [
pd.DataFrame(
(np.concatenate((var_w.data[0], var_e.data[0]),
axis=len(var_w.data[0].shape) - 1).reshape(
nlev_dict[var_w.name], ncoord).T),
columns=[
"{}{}".format(var_w.name, n)
for n in range(nlev_dict[var_w.name])
],
) for var_w, var_e in zip(dataset_w.values(), dataset_e.values())
]
# 4. Concatenate the DataFrames for each var to obtain a single DataFrame
# with (nlat*nlon, nlev*nvar) Note every var can have a different number
# of levels (either pressure levels or height above ground levels)
return pd.concat(var_data, axis=1)
qpf_datetime_list = tidy_ndfd_var_dates(var_lines_list=qpf_lines)
# %%
print(pop12_datetime_list)
print(qpf_datetime_list)
# %%
test = min(pop12_datetime_list)
test_str = dt.datetime.strftime(test, "%Y%m%d%H%M")
test_str
# %%
#https://www.ncdc.noaa.gov/thredds/catalog/ndfd/file/catalog.html #catalog
#temp_data_url = "https://www.ncdc.noaa.gov/thredds/dodsC/ndfd/file/" + "201808/20180822/" + "YDCZ98_KWBN_" + test_str + ".dods"
temp_data_url = "https://www.ncdc.noaa.gov/thredds/dodsC/ndfd/file/" + "201808/20180801/" + "YICZ98_KWBN_201808010446.dods"
temp_ndfd_data = open_dods(temp_data_url)
temp_ndfd_data
# %%
temp_x = temp_ndfd_data['x'][:]
temp_y = temp_ndfd_data['y'][:]
#temp_pop12 = temp_ndfd_data['Total_precipitation_surface_12_Hour_Accumulation_probability_above_0p254'] # pop12
temp_qpf = temp_ndfd_data[
'Total_precipitation_surface_6_Hour_Accumulation'] # qpf
# save pop12 dimentions
#temp_pop12_dims = temp_pop12.dimensions
#temp_pop12_time_dim = temp_pop12_dims[0]
# save qpf dimentions
temp_qpf_dims = temp_qpf.dimensions
#!/usr/bin/env python
from pydap.client import open_dods
import numpy as np
import argparse as ap
parser = ap.ArgumentParser(
description='Use pydap client to access HDF5 files via Hyrax ')
parser.add_argument(
'url',
nargs='?',
default=
'https://eosdap.hdfgroup.org:8080/opendap/data/hdf5/grid_1_2d.h5.dods?temperature[0:1:1][0:1:1]',
help='The URL to access the HDF5 files via Hyrax')
#Change the URL to your own server's address
args = parser.parse_args()
input_url = ''.join(args.url)
#Obtain the data via pydap
dataset = open_dods(input_url)
data = []
data = dataset.temperature[:]
#Generate the output file name from URL
last_slash_index = input_url.rfind('/')
h5_suffix_index = input_url.find(".h5", last_slash_index)
output_string = input_url[last_slash_index + 1:h5_suffix_index]
#Save the data with numpy savetxt
np.savetxt(output_string + '_data.txt', data)
def save_dataset(
fname,
date,
hour,
var_conf,
res,
step,
time_tuple,
lev_idx,
lat_tuple,
lon_tuple,
verbose=False,
):
request = URL.format(
date=date,
hour=hour,
res="{0:.2f}".format(res).replace(".", "p"),
step="" if step == 3 else "_{:1d}hr".format(step),
)
if verbose:
print(request + "lat,lon")
try:
coord = open_dods(request + "lat,lon")
except:
raise OpenFileError("file '{}' not available".format(request[:-1]))
# We don't get the time array from the server since it is in seconds from a
# date. Instead we compute the times in hours manually.
time = range1(*time_tuple, step=step)
# TODO: there is a possible problem here if the division is not exact
time_idx = (int(time_tuple[0] / step), int(time_tuple[1] / step))
# Slicing [:] downloads the data from the server
lat, lon = coord["lat"][:].data, coord["lon"][:].data
# Transform longitudes from range 0..360 to -180..180
lon = np.where(lon > 180, lon - 360, lon)
# Transform into python lists to use the index() method
# TODO: change to find the closest lat/lon with argmin and not an exact match
lat_list, lon_list = lat.tolist(), lon.tolist()
try:
lat_idx = (lat_list.index(lat_tuple[0]), lat_list.index(lat_tuple[1]))
except:
raise ValueError("Latitude not in the grid", lat_tuple)
lat = lat[range1(*lat_idx)].tolist()
if lon_tuple[0] < 0 and lon_tuple[1] > 0:
try:
lon_idx_w = (lon_list.index(lon_tuple[0]), len(lon_list) - 1)
lon_idx_e = (0, lon_list.index(lon_tuple[1]))
except:
raise ValueError("Longitude not in the grid", lon_tuple)
lon_w = lon[range1(*lon_idx_w)].tolist()
lon_e = lon[range1(*lon_idx_e)].tolist()
param_w = {
"lat": lat_idx,
"lon": lon_idx_w,
"time": time_idx,
"lev": lev_idx
}
param_e = {
"lat": lat_idx,
"lon": lon_idx_e,
"time": time_idx,
"lev": lev_idx
}
try:
data_w = get_file(request,
param_w,
var_conf,
time,
lat,
lon_w,
verbose=verbose)
data_e = get_file(request,
param_e,
var_conf,
time,
lat,
lon_e,
verbose=verbose)
except:
raise
data = pd.concat((data_w, data_e), axis=0)
else:
try:
lon_idx = (lon_list.index(lon_tuple[0]),
lon_list.index(lon_tuple[1]))
except:
raise ValueError("Longitude not in the grid", lon_tuple)
lon = lon[range1(*lon_idx)].tolist()
param = {
"lat": lat_idx,
"lon": lon_idx,
"time": time_idx,
"lev": lev_idx
}
try:
data = get_file(request,
param,
var_conf,
time,
lat,
lon,
verbose=verbose)
except:
raise
data.to_csv(fname, sep=" ", float_format="%.3f")
def test_open_dods_with_attributes(self):
dataset = open_dods('http://localhost:8001/.dods', True)
self.assertEqual(dataset.attributes['NC_GLOBAL'], {})
self.assertEqual(dataset.attributes['DODS_EXTRA'], {})
self.assertEqual(dataset.attributes['type'], "Drifters")
def test_open_dods(self):
dataset = open_dods('http://localhost:8001/.dods')
data = np.array(np.rec.fromrecords(DATA, names=dataset.Drifters.keys()))
np.testing.assert_array_equal(data, dataset.Drifters.data)
self.assertEqual(dataset.attributes, {})
# Example (no data level):
#
# *NO0042G.Hg_mon.IMG.air.mercury.1h.NO01L_tekran_42_dup.NO01L_afs..*
#
# Ends up in the following request:
# http://dev-ebas-pydap.nilu.no/NO0042G.Hg_mon.IMG.air.mercury.1h.NO01L_tekran_42_dup.NO01L_afs..dods
# **We will do a quick example with mercury data from the Zeppelin station**
#
# We access the data using the OPeNDAP protocol, we send a dods request and recieve binary data.
#
# This is now parsed by the PyDAP library.
# In[4]:
mercury_ds = open_dods('http://dev-ebas-pydap.nilu.no/NO0042G.Hg_mon.IMG.air.mercury.1h.NO01L_tekran_42_dup.NO01L_afs..dods')
# We get the keys
keys = mercury_ds.keys()
print(keys)
# **Now we can append the data to some variables**
# In[5]:
mercury = mercury_ds['mercury']
# Let's have a quicklook at the data
print(mercury.keys())
