def get_neracoos_wind_data(url,id_s,id_e,id_max_url): #get wind data from neracoos.
url1=url+'wind_speed[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0],wind_direction[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0]'
database_s=open_url(url1)['wind_speed'][int(id_s):int(id_e)]
database_d=open_url(url1)['wind_direction'][int(id_s):int(id_e)]
#lat=database_s['lat']
#lat=round(lat[0],2)
#lon=database_s['lon']
#lon=round(lon[0],2)
depth=database_s['wind_depth']
period=database_s['time']
speed=database_s['wind_speed']
speed=speed[0:].tolist()
period=num2date(period[0:]+date2num(dt.datetime(1858, 11, 17, 0, 0)))
direction=database_d['wind_direction']
direction=direction[0:].tolist()
period_str,wind_all=[],[]
for i in range(len(period)): #convert format to list
period_str.append(dt.datetime.strftime(period[i],'%Y-%m-%d-%H-%M'))
wind_all.append([round(depth[0],1),round(speed[i][0][0][0],2),round(direction[i][0][0][0],2)])
wind,direction=[],[] # figure out bad data and delete
for i in range(len(wind_all)):
wind.append(wind_all[i][1])
direction.append(wind_all[i][2])
id_bad=ml.find((np.array(wind)>300) | (np.array(wind)<-1) | (np.array(direction)<0)| (np.array(direction)>360))
#print id_bad
id_bad=list(id_bad)
id_bad.reverse()
for m in id_bad:
del period_str[m]
del wind_all[m]
return period_str,wind_all
def getcodar_ctl_id(model_option,url,datetime_wanted):
if model_option=='1':
dtime=open_url(url+'?time')
dd=dtime['time']
#print "This option has data from "+str(num2date(dd[0]+date2num(datetime.datetime(2009, 1, 1, 0, 0))))+" to "+str(num2date(dd[-1]+date2num(datetime.datetime(2009, 1, 1, 0, 0))))
print 'This option has data from '+dd[0].strftime("%B %d, %Y")+' to '+dd[-1] .strftime("%B %d, %Y")
id=datetime_wanted-date2num(datetime.datetime(2009, 1, 1, 0, 0))
id=str(int(id))
if model_option=='6':
dtime=open_url(url+'?time')
dd=dtime['time']
ddd=[]
#print 'This option has data from '+dd[0].strftime("%B %d, %Y")+' to '+dd[-1] .strftime("%B %d, %Y")
id=datetime_wanted-date2num(datetime.datetime(2006, 1, 1, 0, 0))
id=str(int(id))
else:
dtime=open_url(url+'?time')
dd=dtime['time']
ddd=[]
for i in list(dtime['time']):
i=round(i,7)
ddd.append(i)
#print "This option has data from "+str(num2date(dd[0]+date2num(datetime.datetime(2001, 1, 1, 0, 0))))+" to "+str(num2date(dd[-1]+date2num(datetime.datetime(2001, 1, 1, 0, 0))))
#print 'This option has data from '+num2date(dd[0]).strftime("%B %d, %Y")+' to '+num2date(dd[-1]).strftime("%B %d, %Y")
id=ml.find(np.array(ddd)==round(datetime_wanted-date2num(datetime.datetime(2001, 1, 1, 0, 0)),7))
for i in id:
id=str(i)
#print 'codar id is '+id
return id
def getsst(second):
#get the index of second from the url
time_tuple = time.gmtime(second)#calculate the year from the seconds
year = time_tuple.tm_year
if year < 1999 or year > 2010:
print 'Sorry there might not be available data for this year'
# WARNING: As of Jan 2012, this data is only stored for 1999-2010
url1 = 'http://tashtego.marine.rutgers.edu:8080/thredds/dodsC/cool/avhrr/bigbight/' + str(year) + '?time[0:1:3269]'
dataset1 = open_url(url1)
times = list(dataset1['time'])
# find the nearest image index
index_second = int(round(np.interp(second, times, range(len(times)))))
#get sst, time, lat, lon from the url
url = 'http://tashtego.marine.rutgers.edu:8080/thredds/dodsC/cool/avhrr/bigbight/' + \
str(year) + '?lat[0:1:1221],lon[0:1:1182],' + \
'mcsst[' + str(index_second) + ':1:' + str(index_second) + \
'][0:1:1221][0:1:1182]' + \
',time[' + str(index_second) + \
':1:' + str(index_second) + ']'
try:
dataset = open_url(url)
except:
print "Please check your url! Cannot access dataset."
sys.exit(0)
sst = dataset['mcsst'].mcsst
time1 = dataset['time']
lat = dataset['lat']
lon = dataset['lon']
return sst, time1, lat, lon
def _load(self, filename, elements, debug=False):
"""Loads data from *.nc, *.p and OpenDap url"""
# Loading pickle file
if filename.endswith(".p"):
f = open(filename, "rb")
data = pkl.load(f)
self._origin_file = data["Origin"]
self.History = data["History"]
if debug:
print "Turn keys into attributs"
self.Grid = ObjectFromDict(data["Grid"])
self.Variables = ObjectFromDict(data["Variables"])
try:
if self._origin_file.startswith("http"):
# Look for file through OpenDAP server
print "Retrieving data through OpenDap server..."
self.Data = open_url(data["Origin"])
# Create fake attribut to be consistent with the rest of the code
self.Data.variables = self.Data
else:
self.Data = self._load_nc(data["Origin"])
except: # TR: need to precise the type of error here
print "the original *.nc file has not been found"
pass
# Loading netcdf file
elif filename.endswith(".nc"):
if filename.startswith("http"):
# Look for file through OpenDAP server
print "Retrieving data through OpenDap server..."
self.Data = open_url(filename)
# Create fake attribut to be consistent with the rest of the code
self.Data.variables = self.Data
else:
# Look for file locally
print "Retrieving data from " + filename + " ..."
self.Data = self._load_nc(filename)
# Metadata
text = "Created from " + filename
self._origin_file = filename
self.History = [text]
# Calling sub-class
print "Initialisation..."
try:
self.Grid = _load_grid(self.Data, elements, self.History, debug=self._debug)
self.Variables = _load_var(self.Data, elements, self.Grid, self.History, debug=self._debug)
except MemoryError:
print "---Data too large for machine memory---"
print "Tip: use ax or tx during class initialisation"
print "--- to use partial data"
raise
elif filename.endswith(".mat"):
raise PyseidonError("---Functionality not yet implemented---")
else:
raise PyseidonError("---Wrong file format---")
def get_neracoos_current_data(url,id_s,id_e,id_max_url): #get wind data from neracoos.
url1=url+'current_speed[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0],current_direction[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0],current_u[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0],current_v[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0]'
database_s=open_url(url1)['current_speed'][int(id_s):int(id_e)]
database_d=open_url(url1)['current_direction'][int(id_s):int(id_e)]
database_u=open_url(url1)['current_u'][int(id_s):int(id_e)]
database_v=open_url(url1)['current_v'][int(id_s):int(id_e)]
#lat=database_s['lat']
#lat=round(lat[0],2)
#lon=database_s['lon']
#lon=round(lon[0],2)
period=database_s['time']
speed=database_s['current_speed']
speed=speed[0:].tolist()
period=num2date(period[0:]+date2num(dt.datetime(1858, 11, 17, 0, 0)))
direction=database_d['current_direction']
direction=direction[0:].tolist()
u=database_u['current_u']
u=u[0:].tolist()
v=database_v['current_v']
v=v[0:].tolist()
period_str,current_all=[],[]
for i in range(len(period)): #convert format to list
period_str.append(dt.datetime.strftime(period[i],'%Y-%m-%d-%H-%M'))
current_all.append([round(speed[i][0][0][0],2),round(direction[i][0][0][0],2),round(u[i][0][0][0],2),round(v[i][0][0][0],2)])
current,u,v,direction=[],[],[],[]# figure out bad data and delete
for i in range(len(current_all)):
current.append(current_all[i][0])
direction.append(current_all[i][1])
u.append(current_all[i][2])
v.append(current_all[i][3])
id_bad=ml.find((np.array(current)>200) | (np.array(current)<-1)|(np.array(direction)<0)| (np.array(direction)>360)|(np.array(u)<-200)| (np.array(u)>200)|(np.array(v)<-200)| (np.array(v)>200))
#print id_bad
id_bad=list(id_bad)
id_bad.reverse()
for m in id_bad:
del period_str[m]
del current_all[m]
return period_str,current_all
def station_info(station_id):
from pydap.client import open_url
url1 = 'http://dods.ndbc.noaa.gov/thredds/dodsC/data/stdmet/'+station_id+'/'+station_id+'h9999.nc'
#url1 = 'http://dods.ndbc.noaa.gov/thredds/dodsC/data/ocean/'+station_id+'/'+station_id+'o9999.nc' #Ocean Dta
url2 = 'http://dods.ndbc.noaa.gov/thredds/dodsC/data/stdmet/'+station_id+'/'+station_id+'.ncml'
#url2=''
try:
dataset = open_url(url1)
except:
try:
dataset = open_url(url2)
except:
print 'OPENDAP url not found: ' + station_id
return False
return station_info_details(station_id,dataset)
def test_timeout(sequence_type_data):
"""Test that timeout works properly"""
TestDataset = DatasetType('Test')
TestDataset['sequence'] = sequence_type_data
TestDataset['byte'] = BaseType('byte', 0)
application = BaseHandler(TestDataset)
# Explictly add latency on the devel server
# to guarantee that it timeouts
def wrap_mocker(func):
def mock_add_latency(*args, **kwargs):
time.sleep(1e-1)
return func(*args, **kwargs)
return mock_add_latency
application = wrap_mocker(application)
with LocalTestServer(application) as server:
url = ("http://0.0.0.0:%s/" % server.port)
# test open_url
assert open_url(url) == TestDataset
with pytest.raises(HTTPError) as e:
open_url(url, timeout=1e-5)
assert 'Timeout' in str(e)
# test open_dods
with pytest.raises(HTTPError):
open_dods(url + '.dods?sequence', timeout=1e-5)
assert 'Timeout' in str(e)
# test sequenceproxy
dataset = open_url(url)
seq = dataset['sequence']
assert isinstance(seq.data, SequenceProxy)
# Change the timeout of the sequence proxy:
seq.data.timeout = 1e-5
with pytest.raises(HTTPError) as e:
next(seq.iterdata())
assert 'Timeout' in str(e)
# test baseproxy:
dat = dataset['byte']
assert isinstance(dat.data, BaseProxy)
# Change the timeout of the baseprox proxy:
dat.data.timeout = 1e-5
with pytest.raises(HTTPError) as e:
dat[:]
assert 'Timeout' in str(e)
def getemolt_sensor(mindtime1,maxdtime1,i_mindepth,i_maxdepth,site2,mindtime,maxdtime):
#According to the conditions to select data from "emolt_sensor"
url2="http://gisweb.wh.whoi.edu:8080/dods/whoi/emolt_sensor?emolt_sensor.SITE,emolt_sensor.TIME_LOCAL,emolt_sensor.YRDAY0_LOCAL,emolt_sensor.TEMP,emolt_sensor.DEPTH_I&emolt_sensor.TIME_LOCAL>="+str(mindtime1)+"&emolt_sensor.TIME_LOCAL<="\
+str(maxdtime1)+"&emolt_sensor.DEPTH_I>="+str(i_mindepth)+"&emolt_sensor.DEPTH_I<="+str(i_maxdepth)+site2
try:
dataset1=open_url(url2)
except:
print 'Sorry, '+url2+' not available'
sys.exit(0)
emolt_sensor=dataset1['emolt_sensor']
try:
sites2=list(emolt_sensor['SITE'])
except:
print "'Sorry, According to your input, here are no value. please check it! ' "
sys.exit(0)
#sites2=list(emolt_sensor['SITE'])
time=list(emolt_sensor['TIME_LOCAL'])
yrday0=list(emolt_sensor['YRDAY0_LOCAL'])
temp=list(emolt_sensor['TEMP'])
depth1=list(emolt_sensor['DEPTH_I'])
time1,temp1,yrday01,sites1,depth=[],[],[],[],[]
for m in range(len(time)):
#if mindtime<=dt.datetime.strptime(str(time[m]),'%Y-%m-%d')<=maxdtime:
if date2num(mindtime)<=yrday0[m]%1+date2num(dt.datetime.strptime(str(time[m]),'%Y-%m-%d'))<=date2num(maxdtime):
#if str(time[m])=='2012-01-01':
temp1.append(temp[m])
yrday01.append(yrday0[m]%1+date2num(dt.datetime.strptime(str(time[m]),'%Y-%m-%d')))
sites1.append(sites2[m])
time1.append(date2num(dt.datetime.strptime(str(time[m]),'%Y-%m-%d')))
depth.append(depth1[m])
#print len(temp1)
return time1,yrday01,temp1,sites1,depth,
def test_variable_esgf_query(self):
assert(os.environ.get('OPENID_ESGF'))
assert(os.environ.get('PASSWORD_ESGF'))
session = esgf.setup_session(os.environ.get('OPENID_ESGF'),
os.environ.get('PASSWORD_ESGF'),
check_url=self.url)
# Ensure authentication:
res = pydap.net.follow_redirect(self.test_url, session=session)
assert(res.status_code == 200)
dataset = open_url(self.url, session=session, output_grid=False)
data = dataset['pr'][0, 200:205, 100:105]
expected_data = [[[5.23546005e-05, 5.48864300e-05,
5.23546005e-05, 6.23914966e-05,
6.26627589e-05],
[5.45247385e-05, 5.67853021e-05,
5.90458621e-05, 6.51041701e-05,
6.23914966e-05],
[5.57906533e-05, 5.84129048e-05,
6.37478297e-05, 5.99500854e-05,
5.85033267e-05],
[5.44343166e-05, 5.45247385e-05,
5.60619228e-05, 5.58810752e-05,
4.91898136e-05],
[5.09982638e-05, 4.77430549e-05,
4.97323490e-05, 5.43438946e-05,
5.26258664e-05]]]
assert(np.isclose(data, expected_data).all())
def wrapper(*args, **kwargs):
url, varname, bbox, dt = fetch(*args, **kwargs)
ds = open_url(url)
for var in ds.keys():
if var.lower().startswith("lon") or var.lower() == "x":
lonvar = var
if var.lower().startswith("lat") or var.lower() == "y":
latvar = var
if var.lower().startswith("time") or var.lower() == "t":
timevar = var
lat = ds[latvar][:].data
lon = ds[lonvar][:].data
lon[lon > 180] -= 360
res = abs(lat[0]-lat[1]) # assume rectangular grid
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
t = ds[timevar]
tt = netcdf4.num2date(t[:].data, units=t.units)
ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]]
if len(ti) > 0:
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
if len(ds[varname].data[0].shape) > 3:
data = ds[varname].data[0][ti[0]:ti[-1]+1, 0, lati[0]:lati[-1]+1, loni[0]:loni[-1]+1]
else:
data = ds[varname].data[0][ti[0]:ti[-1]+1, 0, lati[0]:lati[-1]+1, loni[0]:loni[-1]+1]
dt = tt[ti]
else:
data = None
dt = None
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
return data, lat, lon, dt
def test_parse_constraints_boolean(self):
test_data = np.asanyarray([True,False,True,True,False])
self.cov.set_parameter_values('boolean',value=test_data)
dataset = open_url(self.request_url)
result = []
result = np.asanyarray([d for d in dataset['data']['boolean']])
self.assertTrue(np.array_equal(result, test_data))
def load(self,fldname, **kwargs):
""" Load velocity fields for a given day"""
self._timeparams(**kwargs)
if fldname == "uv":
self.load('u',**kwargs)
self.load('v',**kwargs)
self.uv = np.sqrt(self.u[:,1:,:]**2 + self.v[:,:,1:]**2)
return
if self.opendap:
tpos = int(self.jd) - 714800
self.k1 = kwargs.get("k1", getattr(self, "k1", self.klev))
self.k2 = kwargs.get("k2", getattr(self, "k2", k1+1))
dapH = open_url(self.dapurl)
fld = dapH[fldname][tpos,self.k1:self.k2,
self.j1:self.j2,self.i1:self.i2]
else:
filename = self.jd2filename(self.jd)
if not os.path.isfile(filename):
print "File missing"
url = urlparse.urljoin(self.dataurl,os.path.basename(filename))
self.retrive_file(url, filename)
with Dataset(filename) as nc:
nc.set_auto_mask(False)
fld = nc.variables[fldname][:,self.k1:self.k2,
self.j1:self.j2,
self.i1:self.i2].copy()
self.ssh = np.squeeze(nc.variables['zeta'][:])
self.zlev = ((self.depth + self.ssh)[np.newaxis,:,:] *
self.Cs_r[:,np.newaxis,np.newaxis])
fld[fld>9999] = np.nan
setattr(self, fldname, np.squeeze(fld))
def test_lazy_evaluation_getattr(self):
"""Test that the dataset is only loaded when accessed."""
original = open_url('/', application=self.app)
dataset = original.functions.mean(original.SimpleGrid, 0)
self.assertIsNone(dataset.dataset)
dataset.SimpleGrid
self.assertIsNotNone(dataset.dataset)
def test_Functions(self):
dataset = open_url('http://localhost:8001/')
rain = dataset.rain
self.assertEqual(rain.rain.shape, (2, 3))
functions = Functions('http://localhost:8001/')
dataset = functions.mean(rain, 0)
self.assertEqual(dataset.rain.rain.shape, (3,))
np.testing.assert_array_equal(dataset.rain.rain.data,
np.array([1.5, 2.5, 3.5]))
dataset = functions.mean(rain, 0)
self.assertEqual(dataset['rain']['rain'].shape, (3,))
np.testing.assert_array_equal(dataset.rain.rain.data,
np.array([1.5, 2.5, 3.5]))
dataset = functions.mean(rain, 1)
self.assertEqual(dataset.rain.rain.shape, (2,))
np.testing.assert_array_equal(dataset.rain.rain.data,
np.array([1.0, 4.0]))
dataset = functions.mean(rain, 1)
self.assertEqual(dataset['rain']['rain'].shape, (2,))
np.testing.assert_array_equal(dataset.rain.rain.data,
np.array([1.0, 4.0]))
dataset = functions.mean(functions.mean(rain, 0), 0)
self.assertEqual(dataset['rain']['rain'].shape, ())
np.testing.assert_array_equal(dataset.rain.rain.data,
np.array(2.5))
def fetch(cls, asset, tile, date):
""" Get this asset for this tile and date (using OpenDap service) """
url = cls._assets[asset].get('url', '') % (date.year, tile, date.year)
source = cls._assets[asset]['source']
loc = "%s/%s" % (url, source)
print loc
dataset = open_url(loc)
x0 = dataset['x'].data[0] - 500.0
y0 = dataset['y'].data[0] + 500.0
day = date.timetuple().tm_yday
iday = day - 1
data = np.array(dataset[asset][iday, :, :]).squeeze().astype('float32')
ysz, xsz = data.shape
description = cls._assets[asset]['description']
meta = {'ASSET': asset, 'TILE': tile, 'DATE': str(date.date()), 'DESCRIPTION': description}
sday = str(day).zfill(3)
fout = os.path.join(cls.Repository.path('stage'), "daymet_%s_%s_%4d%s.tif" % (asset, tile, date.year, sday))
geo = [float(x0), cls._defaultresolution[0], 0.0, float(y0), 0.0, -cls._defaultresolution[1]]
geo = np.array(geo).astype('double')
dtype = create_datatype(data.dtype)
imgout = gippy.GeoImage(fout, xsz, ysz, 1, dtype)
imgout.SetBandName(asset, 1)
imgout.SetNoData(-9999.)
imgout.SetProjection(PROJ)
imgout.SetAffine(geo)
imgout[0].Write(data)
def __init__(self, url):
""" Constructor, reads in a file from the url provided
@param url: OpenNDAP url for the NetCDF file
"""
self.url = url
# Let's open the file using OpenNDAP
f = open_url(url)
# Pull out the attributes, and column names
self.attributes = f.attributes['NC_GLOBAL']
self.columns = f.keys()
if 'time' in f:
# There's a time dimension in this dataset, so let's grab the possible
# values out
try:
time_units = f['time'].attributes['units']
self.time_values = [parse(t, time_units) for t in f['time']]
except IndexError:
# Not a parseable time format
pass
del f
def load(url, variable):
'''Load a Dataset from an OpenDAP URL
:param url: The OpenDAP URL for the dataset of interest.
:type url: String
:param variable: The name of the variable to read from the dataset.
:type variable: String
:returns: A Dataset object containing the dataset pointed to by the
OpenDAP URL.
:raises: ServerError
'''
# Grab the dataset information and pull the appropriate variable
d = open_url(url)
dataset = d[variable]
# Grab the lat, lon, and time variable names.
# We assume the variable order is (time, lat, lon)
dataset_dimensions = dataset.dimensions
time = dataset_dimensions[0]
lat = dataset_dimensions[1]
lon = dataset_dimensions[2]
# Time is given to us in some units since an epoch. We need to convert
# these values to datetime objects. Note that we use the main object's
# time object and not the dataset specific reference to it. We need to
# grab the 'units' from it and it fails on the dataset specific object.
times = np.array(_convert_times_to_datetime(d[time]))
lats = np.array(dataset[lat][:])
lons = np.array(dataset[lon][:])
values = np.array(dataset[:])
return Dataset(lats, lons, times, values, variable)
def get_data_from_opendap(self, x, y, start=None, end=None):
"""
Return list of dicts for data at x, y.
Start, end are datetimes, and default to the first and last
datetime in the file.
"""
try:
dataset = client.open_url(self.url)
except ServerError:
return []
index_start = 0
if start is not None:
index_start = self.index(start)
if end is None:
index_end = self.timesteps - 1
else:
index_end = self.index(end)
precipitation = dataset['precipitation']['precipitation']
tuples = zip(
iter(self._get_datetime_generator(start=index_start,
end=index_end)),
precipitation[y, x, index_start: index_end + 1][0, 0, :],
)
return [dict(unit='mm/5min', datetime=d, value=float(p))
for d, p in tuples
if not p == config.NODATAVALUE]
def read_opendap_index(date=None,domain=None):
url = parse_url(date)
print url[0]
dataset = open_url(url[0])
lats = dataset['lat'][:]
lons = dataset['lon'][:]
pres = dataset['pressure'][:]/100. #[hPa]
lats_idx = np.where((lats>domain['latn'])&
(lats<domain['latx']))[0]
lons_idx = np.where((lons>360+domain['lonn'])&
(lons<360+domain['lonx']))[0]
if domain['preslvl'] is not None:
pres_idx = np.where(pres==domain['preslvl'])[0][0]
pres = pres[pres_idx]
else:
pres_idx = None
last = lats_idx[0]
laen = lats_idx[-1]+1
lost = lons_idx[0]
loen = lons_idx[-1]+1
latsnew = lats[lats_idx]
lonsnew = lons[lons_idx]-360
index={'last':last,'laen':laen,'lost':lost,'loen':loen,'plvl':pres_idx}
coords={'lats':latsnew,'lons':lonsnew,'pres':pres}
return index,coords
def get_coors(modelname, lo, la, lonc, latc, lon, lat, siglay, h, depth,startrecord, endrecord):
if lo>90:
[la,lo]=dm2dd(la,lo)
print 'la, lo',la, lo
latd,lond=[la],[lo]
# kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face
# kv,distanceV=nearlonlat(lon,lat,lo,la)
kf,distanceF = nearest_point_index(lo,la,lonc,latc,num=1)
kv,distanceV = nearest_point_index(lo,la,lon,lat,num=1)
kf = kf[0][0]
kv = kv[0][0]
print 'kf:', kf
if h[kv] < 0:
print 'Sorry, your position is on land, please try another point'
sys.exit()
depthtotal=siglay[:,kv]*h[kv]
layer=np.argmin(abs(depthtotal-depth))
for i in range(startrecord,endrecord):# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
############read the particular time model from website#########
# print 'la, lo, i', la, lo, i
timeurl='['+str(i)+':1:'+str(i)+']'
uvposition=str([layer])+str([kf])
data_want = ('u'+timeurl+uvposition, 'v'+timeurl+uvposition)
# if urlname=="30yr":
# url='http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3?'+\
# 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition
# elif urlname == "GOM3":
# url="http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc?"+\
# 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition
# else:
# url="http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc?"+\
# 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition
url = url_with_time_position(modelname, data_want)
dataset = open_url(url)
u=np.array(dataset['u'])
v=np.array(dataset['v'])
print 'u, v, i', u[0,0,0], v[0,0,0],i
################get the point according the position###################
par_u=u[0,0,0]
par_v=v[0,0,0]
xdelta=par_u*60*60 #get_coors
ydelta=par_v*60*60
latdelta=ydelta/111111
londelta=(xdelta/(111111*np.cos(la*np.pi/180)))
la=la+latdelta
lo=lo+londelta
latd.append(la)
lond.append(lo)
# kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face
# kv,distanceV=nearlonlat(lon,lat,lo,la)# nearest triangle vertex V - vertex
kf,distanceF = nearest_point_index(lo,la,lonc,latc,num=1)
kv,distanceV = nearest_point_index(lo,la,lon,lat,num=1)
kf, kv = kf[0][0], kv[0][0]
depthtotal=siglay[:,kv]*h[kv]
# layer=np.argmin(abs(depthtotal-depth))
if distanceV>=0.3:
if i==startrecord:
print 'Sorry, your start position is NOT in the model domain'
break
return latd ,lond
def get_last_forecast(self,url):
now = datetime.datetime.now(pytz.utc)
# it takes 5.5 hours before data arrive on GrADS site
last = now + datetime.timedelta(hours = -6)
for _ in range(6):
# forecasts are on h=[0,6,12]
h = min(int(last.hour/6),2)*6
last = last.replace(hour = h)
hour = '%02d' % last.hour
date = now.strftime('%Y%m%d')
url = url.format(date=date,hour=hour)
try:
logger.debug('querying '+url)
dataset = open_url(url)
logger.debug('Forecast found for date={date}, hour={hour}'.format(date=date,hour=hour))
return dataset
except Exception as e:
logger.warning('Forecast not found: %s' % e)
# try previous forecast
last = last + datetime.timedelta(hours = -6)
logger.error('No GEFS forecast found')
return None
def fetch_data(URL):
cnt = 0
while True:
if cnt > 10:
break
else:
try:
dataset = open_url(URL)
var = dataset['precipitation']
lon = np.array(dataset['nlon'])
lat = np.array(dataset['nlat'])
ind_LonMin = int(np.argmin(abs(lon-float(xmin))))
ind_LonMax = int(np.argmin(abs(lon-float(xmax))))
ind_LatMin = int(np.argmin(abs(lat-float(ymin))))
ind_LatMax = int(np.argmin(abs(lat-float(ymax))))
Data = var[ind_LonMin:ind_LonMax, ind_LatMin:ind_LatMax]
LLX = lon[ind_LonMin]
LLY = lat[ind_LatMin]
except:
cnt = cnt + 1
print 'fetch atempt ', cnt
continue
break
d = np.flipud(Data.T)
return d, LLX, LLY
def pic_trend(lond, latd):
dt=60*60.
tau=dt/111111.
lont=[]
latt=[]
ufinal=[]
vfinal=[]
for i in range(startrecord,endrecord):
timeurl = '['+str(i)+':1:'+str(i)+']'
uvposition = str([0])+'[0:1:90414]' # this is the number of grid points in thie 30yr model
url = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3?'+'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition
dataset = open_url(url)
utotal=np.array(dataset['u'])
vtotal=np.array(dataset['v'])
times=np.array(dataset['Times'])
u=utotal[0,0,:]
v=vtotal[0,0,:]
lont.append(lond)
latt.append(latd)
lond,latd,uinterplation,vinterplation=RungeKutta4_lonlat(lond,latd,Grid,u,v,tau)
ufinal.append(uinterplation)
vfinal.append(vinterplation)
kv,distance=nearlonlat(lon,lat,lond,latd)
if distance>=0.3:
break
fig=plt.figure()
Q=plt.quiver(lont,latt,ufinal,vfinal,scale=5.)
plt.show()
def opendap_fetch(cls, asset, date):
""" Get array proxy from OpenDap for this asset and date """
url = cls._assets[asset].get('url', '')
if url == '':
raise Exception("%s: URL not defined for asset %s" % (cls.__name__, asset))
success = False
for ver in ['100', '200', '300', '301', '400']:
if asset != "FRLAND":
f = cls._assets[asset]['source'] % (ver, date.year, date.month, date.day)
loc = "%s/%04d/%02d/%s" % (url, date.year, date.month, f)
else:
f = cls._assets[asset]['source'] % (ver, 0, 0, 0)
loc = "%s/1980/%s" % (url, f)
try:
with Timeout(30):
dataset = open_url(loc)
except Timeout.Timeout:
print "Timeout"
except Exception,e:
pass
else:
success = True
break
def get_dataset(url, *labels):
dataset = open_url(url)
data = []
for label in labels:
add = np.array(dataset[label])
data.append(add)
return data
def get_data(var_id, row, col):
base_url = ('http://dapds00.nci.org.au/thredds/dodsC/rr9/Climate/eMAST/'
'ANUClimate/0_01deg/v1m0_aus/mon/land/%s/e_01/1970_2012/' % (var_id))
emast_id = "eMAST_ANUClimate_mon_tmax_v1m0"
start_date = "1970-01-01"
stop_date = "2000-12-31"
current = datetime.strptime(start_date, "%Y-%m-%d")
stop = datetime.strptime(stop_date, "%Y-%m-%d")
tmax = []
dates = []
while current < stop:
if current.month < 10:
month = "0%s" % (current.month)
else:
month = "%s" % (current.month)
year = current.year
url = "%s%s_%s%s.nc" % (base_url, emast_id, year, month)
dataset = open_url(url)
variable = dataset['air_temperature']
#print variable[0,2000:2005,2000:2005].array[:]
tmax.append(variable[0,2000:2005,2000:2005].array[:][0][0][0])
dates.append(current)
current += relativedelta(months=1)
f = open("tmax_%d_%d.txt" % (row, col), "w")
for i in xrange(tmax):
f >> tmax
f.close()
def test_lazy_evaluation_getattr(ssf_app):
"""Test that the dataset is only loaded when accessed."""
original = open_url('/', application=ssf_app)
dataset = original.functions.mean(original.SimpleGrid, 0)
assert dataset.dataset is None
dataset.SimpleGrid
assert dataset.dataset is not None
def get_dataset(url):
try:
dataset = open_url(url)
except:
print "Sorry, " + url + "is not available"
sys.exit(0)
return dataset
def __init__(self, day, month, year, lat, lon, ndays):
self.day = day
self.month = month
self.year = year
self.lat = lat
self.lon = lon
self.ndays = ndays
self.datetime_date = datetime.date(self.year, self.month, self.day)
path = self.construct_path('http://goldsmr2.sci.gsfc.nasa.gov:80/opendap/MERRA/MAT1NXFLX.5.2.0', 'MERRA300.prod.assim.tavg1_2d_flx_Nx.', 0)
dataset = open_url(path)
self.native_lat = dataset['YDim'][:]
self.native_lon = dataset['XDim'][:]
self.lat_idx_native = np.where(np.min(np.abs(self.native_lat - lat)) == np.abs(self.native_lat - lat))[0][0]
self.lon_idx_native = np.where(np.min(np.abs(self.native_lon - lon)) == np.abs(self.native_lon - lon))[0][0]
self.lat_actual = self.native_lat[self.lat_idx_native]
self.lon_actual = self.native_lon[self.lon_idx_native]
return
def get_dataset(url):
try:
dataset = open_url(url)
except:
print 'Sorry, ' + url + 'is not available'
sys.exit(0)
return dataset
def test_verify_open_url(sequence_type_data):
"""Test that open_url raises the correct SSLError"""
warnings.simplefilter("always")
TestDataset = DatasetType('Test')
TestDataset['sequence'] = sequence_type_data
TestDataset['byte'] = BaseType('byte', 0)
application = BaseHandler(TestDataset)
with LocalTestServerSSL(application, ssl_context='adhoc') as server:
try:
open_url(server.url, verify=False, session=requests.Session())
except (ssl.SSLError, requests.exceptions.SSLError):
pytest.fail("SSLError should not be raised.")
with pytest.raises(requests.exceptions.SSLError):
open_url(server.url, session=requests.Session())
if not (sys.version_info >= (3, 0) and sys.version_info < (3, 4, 4)):
# verify is disabled by default for python 3 before 3.4.4:
with pytest.raises(requests.exceptions.SSLError):
open_url(server.url)
def test_client(self):
dataset = open_url("http://localhost:8001/", application=self.app)
self.assertEqual(self.dataset.keys(), ["foo%5B"])
self.assertEqual(dataset["foo["].name, "foo%5B")
self.assertEqual(dataset["foo%5B"][0], 1)
def test_original(ssf_app):
"""Test an unmodified call, without function calls."""
original = open_url('/', application=ssf_app)
assert (original.SimpleGrid.SimpleGrid.shape == (2, 3))
def __init__(self, urlpath):
self.data = open_url(urlpath)
self._field = None
def PRECIP_DI_CAL(date='2014-06-06',bbox=[-87.5, -31.1, -29.3, 0.1]):
opendap_url_mon='http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/gpcp/precip.mon.mean.nc'
opendap_url_ltm='http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/gpcp/precip.mon.ltm.nc'
# what is the input of the module
logging.info(date)
logging.info(bbox)
# convert iso-date to gregorian calendar and get the month
dta=(dt.datetime.strptime(date,'%Y-%m-%d').date()-dt.datetime.strptime('1800-01-01','%Y-%m-%d').date()).days
mon=(dt.datetime.strptime(date,'%Y-%m-%d').date()).month
# open opendap connection and request the avaialable time + lon/lat
dataset_mon = open_url(opendap_url_mon)
time=dataset_mon.time[:]
lat=dataset_mon.lat[:]
lon=dataset_mon.lon[:]
dt_ind=next((index for index,value in enumerate(time) if value > dta),0)-1
# convert bbox into coordinates and convert OL lon to GPCP lon where needed
minlon = bbox[0]
if minlon < 0: minlon += 360 #+ 180 # GPCP is from 0-360, OL is from -180-180
maxlon = bbox[2]
if maxlon < 0: maxlon += 360 #+ 180 # GPCP is from 0-360, OL is from -180-180
minlat = bbox[1]
maxlat = bbox[3]
lat_sel = (lat>minlat)&(lat<maxlat)
lat_sel[np.nonzero(lat_sel)[0]-1] = True
# ugly method to decide if there are two areas to select
# prepare lon/lat subset arrays
check_if = 0 # If this one is 1, than there are two areas to check
if minlon >= maxlon:
check_if = 1
lon_sel = np.invert((lon<minlon)&(lon>maxlon))
else:
lon_sel = (lon>minlon)&(lon<maxlon)
# request the subset from opendap
dataset_mon=dataset_mon['precip'][dt_ind,lat_sel,lon_sel]
dataset_ltm = open_url(opendap_url_ltm)
dataset_ltm=dataset_ltm['precip'][mon-1,lat_sel,lon_sel]
mon = np.ma.masked_less((dataset_mon['precip'][:]).squeeze(),0)
ltm = np.ma.masked_less((dataset_ltm['precip'][:]).squeeze(),0)
# if two areas make sure the subset is applied appropriate
if check_if == 1:
subset = np.ones((mon.shape[0]), dtype=bool)[None].T * lon_sel
mon = np.roll(mon,
len(lon)/2,
axis=1)[np.roll(subset,
len(lon)/2,
axis=1)].reshape(mon.shape[0],
subset.sum()/mon.shape[0])
ltm = np.roll(ltm,
len(lon)/2,
axis=1)[np.roll(subset,
len(lon)/2,
axis=1)].reshape(ltm.shape[0],
subset.sum()/ltm.shape[0])
# calculate PAP
PAP=(mon-ltm)/(ltm+1)*100
# prepare output for GDAL
papFileName = 'PRECIP_DI'+date +'.tif'
driver = gdal.GetDriverByName( "GTiff" )
ds = driver.Create(papFileName, mon.shape[1], mon.shape[0], 1,gdal.GDT_Int16)
# set projection information
projWKT='GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'
ds.SetProjection(projWKT)
# set geotransform information
geotransform_input = dataset_mon.lon[:]
if check_if == 1:
dataset_mon.lon[:][np.where(dataset_mon.lon[:] > 180)] -= 360
geotransform_input = np.roll(dataset_mon.lon[:], len(lon)/2, axis=0)[np.roll(subset, len(lon)/2, axis=1)[0]]
geotransform = (min(geotransform_input),2.5, 0,max(dataset_mon.lat[:]),0,-2.5)
ds.SetGeoTransform(geotransform)
# write the data
ds.GetRasterBand(1).WriteArray(PAP)
#ds=None
return ds
for i in range(len(bbox['lon'])):
val = bbox['lon'][i]
if val < 0:
bbox['lon'][i] = 180 + val
if bbox['lon'][0] > bbox['lon'][1]: [bbox['lon'][1], bbox['lon'][0]]
print(bbox)
#%%
for v in var:
print(v)
print(str(pytime.time() - start_time) + ' seconds elapsed')
url = url + v
x = open_url(url)
not_var = ['lat', 'lon', 'time', 'time_bnds']
if not init:
init = True
for n in not_var:
if n not in ['time_bnds', 'time']:
dic[n] = x[n][:].data
if n == 'time':
conv_time = []
origin = dt.datetime(1800, 1, 1)
time = x['time'][:].data
for t in time:
t_0 = origin + dt.timedelta(hours=t)
def download(self):
"""
Migrate it to use np.lib.arrayterator.Arrayterator
"""
url_h = "%s/%s-h-daily" % (self.metadata['urlbase'], self.metadata['source_filename'])
dataset_h = open_url(url_h)
url_uv = "%s/%s-uv-daily" % (self.metadata['urlbase'], self.metadata['source_filename'])
dataset_uv = open_url(url_uv)
# ----
if 't_ini' not in self.metadata['limits']:
self.metadata['limits']['t_ini'] = 0
if 't_fin' not in self.metadata['limits']:
self.metadata['limits']['t_fin'] = dataset_h['time'].shape[0]
if 't_step' not in self.metadata['limits']:
self.metadata['limits']['t_step'] = 0
else:
print "Atention!! t_step set to: %s" % self.metadata['limits']['t_step']
t_ini = self.metadata['limits']['t_ini']
t_fin = self.metadata['limits']['t_fin']
t_step = self.metadata['limits']['t_step']
# ----
data={}
#
#from coards import from_udunits
t0=datetime(1950,1,1)
#if (re.match('^hours since \d{4}-\d{2}-\d{2}$',dataset_h['time'].attributes['units'])):
if (re.match('^hours since 1950-01-01',dataset_h['time'].attributes['units'])):
data['datetime']=numpy.array([t0+timedelta(hours=h) for h in dataset_h['time'][t_ini:t_fin:t_step].tolist()])
else:
print "Problems interpreting the time"
return
#time = self.nc.createVariable('time', 'i', ('time',))
#time[:] = dataset_h['time'][t_ini:t_fin:t_step]
#time.units = dataset_h['time'].attributes['units']
#data['time'] = time
#
limits=self.metadata['limits']
Lat=dataset_h['NbLatitudes']
Lon=dataset_h['NbLongitudes']
Latlimits=numpy.arange(Lat.shape[0])[(Lat[:]>=limits["LatIni"]) & (Lat[:]<=limits["LatFin"])]
Latlimits=[Latlimits[0],Latlimits[-1]]
Lonlimits=numpy.arange(Lon.shape[0])[(Lon[:]>=limits["LonIni"]) & (Lon[:]<=limits["LonFin"])]
Lonlimits=[Lonlimits[0],Lonlimits[-1]]
data['Lon'], data['Lat'] = numpy.meshgrid( (Lon[Lonlimits[0]:Lonlimits[-1]]), (Lat[Latlimits[0]:Latlimits[-1]]) )
#------
self.data = data
#Arrayterator = numpy.lib.arrayterator.Arrayterator
#dataset = dataset_h['Grid_0001']['Grid_0001']
#ssh = Arrayterator(dataset)[t_ini:t_fin:t_step]
#blocks = 1e4
file = os.path.join(self.metadata['datadir'],self.metadata['source_filename']+".nc")
nc = pupynere.netcdf_file(file,'w')
nc.createDimension('time', len(range(t_ini,t_fin,t_step)))
nc.createDimension('lon', (Lonlimits[-1]-Lonlimits[0]))
nc.createDimension('lat', (Latlimits[-1]-Latlimits[0]))
dblocks = max(1,int(1e5/((Lonlimits[-1]-Lonlimits[0])*(Latlimits[-1]-Latlimits[0]))))
ti = numpy.arange(t_ini, t_fin, t_step)
blocks = ti[::dblocks]
if ti[-1] not in blocks:
blocks = numpy.append(blocks,t_fin)
ntries = 40
#------
for v, dataset, missing_value in zip(['h','u','v'], [dataset_h['Grid_0001']['Grid_0001'], dataset_uv['Grid_0001']['Grid_0001'], dataset_uv['Grid_0002']['Grid_0002']], [dataset_h['Grid_0001']._FillValue, dataset_uv['Grid_0001']._FillValue, dataset_uv['Grid_0002']._FillValue]):
print "Getting %s" % v
#data['h'] = ma.masked_all((len(ti),Lonlimits[-1]-Lonlimits[0], Latlimits[-1]-Latlimits[0]), dtype=numpy.float64)
self.data[v] = nc.createVariable(v, 'f4', ('time', 'lat', 'lon'))
self.data[v].missing_value = missing_value
for b1, b2 in zip(blocks[:-1], blocks[1:]):
print "From %s to %s of %s" % (b1, b2, blocks[-1])
ind = numpy.nonzero((ti>=b1) & (ti<b2))
for i in range(ntries):
print "Try n: %s" % i
try:
self.data[v][ind] = dataset[b1:b2:t_step, Lonlimits[0]:Lonlimits[-1],Latlimits[0]:Latlimits[-1]].swapaxes(1,2).astype('f')
break
except:
waitingtime = 30+i*20
print "Failed to download. I'll try again in %ss" % waitingtime
time.sleep(waitingtime)
def test_open_url(sequence_app):
"""Open an URL and check dataset keys."""
dataset = open_url('http://localhost:8001/', sequence_app)
assert list(dataset.keys()) == ["cast"]
def getData(station_id, starttime, endtime, outvar):
"""
Access data from the NOAA opendap database.pointNC2shp(ncfile,shpfile)
Usage example:
data,t = getData("8639348","20120501","20120512","conductivity")
Input variables:
station_id: string
starttime: string "yyyymmdd"
endtime: string "yyyymmdd"
outvar: string with variable name
one of: 'waterlevel', 'conductivity', 'watertemp', 'airtemp', 'airpressure',
'windspeed' or 'winddirn'
Output Variables:
data: vector of floats with data
t: time vector (matlab datenum format)
Note that this is not a netcdf file dataserver.
See this website for guidance:
https://oceana.mbari.org/confluence/display/OneStopShopping/Examples+using+pydap+from+Python+to+access+BOG+data+via+DRDS
"""
# Items unique to each data site
baseurl = "http://opendap.co-ops.nos.noaa.gov/dods/IOOS/"
if outvar == 'waterlevel':
# Six minute verified waterlevel data
url = baseurl + "SixMin_Verified_Water_Level"
seqname = 'WATERLEVEL_6MIN_VFD_PX'
varname = 'WL_VALUE'
attribs = {'long_name': 'Water surface elevation', 'units': 'm'}
elif outvar == 'conductivity':
# Conductivity (millisiemens/cm)
url = baseurl + "Conductivity"
seqname = 'CONDUCTIVITY_PX'
varname = 'CONDUCTIVITY'
attribs = {
'long_name': 'Water conductivity',
'units': 'millisiemens/cm'
}
elif outvar == 'watertemp':
# Water temperature (degC)
url = baseurl + "Water_Temperature"
seqname = 'WATER_TEMPERATURE_PX'
varname = 'WaterTemp'
attribs = {'long_name': 'Water temperature', 'units': 'degreesC'}
elif outvar == 'airtemp':
# Air Temperature (degC)
url = baseurl + "Air_Temperature"
seqname = 'AIR_TEMPERATURE_PX'
varname = 'AirTemp'
attribs = {'long_name': ' Air temperature', 'units': 'degreesC'}
elif outvar == 'airpressure':
# Air Presure (millibars)
url = baseurl + "Barometric_Pressure"
seqname = 'BAROMETRIC_PRESSURE_PX'
varname = 'BP'
attribs = {'long_name': 'Air pressure', 'units': 'mb'}
elif outvar == 'windspeed':
# Wind Speed (m/s)
url = baseurl + "Wind"
seqname = 'WIND_PX'
varname = 'Wind_Speed'
attribs = {'long_name': 'Wind speed', 'units': 'm/s'}
elif outvar == 'winddirn':
# Wind Direction (degrees)
url = baseurl + "Wind"
seqname = 'WIND_PX'
varname = 'Wind_Direction'
attribs = {'long_name': 'Wind direction', 'units': 'degrees'}
# Open the database
nc = open_url(url)
#my_station = nc.WATERLEVEL_6MIN_VFD_PX[(nc.WATERLEVEL_6MIN_VFD_PX._STATION_ID == station_id) & \
# (nc.WATERLEVEL_6MIN_VFD_PX._DATUM == "MSL") & \
# (nc.WATERLEVEL_6MIN_VFD_PX._BEGIN_DATE ==starttime) & \
# (nc.WATERLEVEL_6MIN_VFD_PX._END_DATE==endtime)]
print 'Retrieving data '+outvar+' @ site # '+station_id+' for date range: '+\
starttime+' to '+endtime+'...'
try:
# Build a query with the server
if outvar == 'waterlevel':
# Water level requires a datum in the query
my_station = nc[seqname][(nc[seqname]._STATION_ID == station_id) & \
(nc[seqname]._DATUM == "MSL") & \
(nc[seqname]._BEGIN_DATE ==starttime) & \
(nc[seqname]._END_DATE==endtime)]
else:
my_station = nc[seqname][(nc[seqname]._STATION_ID == station_id) & \
(nc[seqname]._BEGIN_DATE ==starttime) & \
(nc[seqname]._END_DATE==endtime)]
print "Query ok - downloading data..."
# Get the data
#data = np.zeros((len(my_station['DATE_TIME']),1))
#t = np.zeros((len(my_station['DATE_TIME']),1))
k = 0
data = []
t = []
for dt, d in zip(my_station['DATE_TIME'], my_station[varname]):
#data[k,0]=np.float(d)
data.append(d)
t.append(parseDate(dt))
k = k + 1
except:
print "The date range and/or the variable: " + varname + " are not available from station #: " + station_id
#data = np.zeros([0,1])
#t=np.zeros([0,1])
data = []
t = []
return data, t, attribs
def _load(self, filename, elements, debug=False):
"""Loads data from *.nc, *.p and OpenDap url"""
#Loading pickle file
if filename.endswith('.p'):
f = open(filename, "rb")
data = pkl.load(f)
self._origin_file = data['Origin']
self.History = data['History']
if debug: print "Turn keys into attributs"
self.Grid = ObjectFromDict(data['Grid'])
self.Variables = ObjectFromDict(data['Variables'])
try:
if self._origin_file.startswith('http'):
#Look for file through OpenDAP server
print "Retrieving data through OpenDap server..."
self.Data = open_url(data['Origin'])
#Create fake attribut to be consistent with the rest of the code
self.Data.variables = self.Data
else:
#WB_Alternative: self.Data = sio.netcdf.netcdf_file(filename, 'r')
#WB_comments: scipy has causes some errors, and even though can be
# faster, can be unreliable
#self.Data = nc.Dataset(data['Origin'], 'r')
self.Data = netcdf.netcdf_file(data['Origin'], 'r',mmap=True)
except: #TR: need to precise the type of error here
print "the original *.nc file has not been found"
pass
#Loading netcdf file
elif filename.endswith('.nc'):
if filename.startswith('http'):
#Look for file through OpenDAP server
print "Retrieving data through OpenDap server..."
self.Data = open_url(filename)
#Create fake attribut to be consistent with the rest of the code
self.Data.variables = self.Data
else:
#Look for file locally
print "Retrieving data from " + filename + " ..."
#WB_Alternative: self.Data = sio.netcdf.netcdf_file(filename, 'r')
#WB_comments: scipy has causes some errors, and even though can be
# faster, can be unreliable
#self.Data = nc.Dataset(filename, 'r')
self.Data = netcdf.netcdf_file(filename, 'r',mmap=True)
#Metadata
text = 'Created from ' + filename
self._origin_file = filename
self.History = [text]
# Calling sub-class
print "Initialisation..."
try:
self.Grid = _load_grid(self.Data,
elements,
self.History,
debug=self._debug)
self.Variables = _load_var(self.Data,
elements,
self.Grid,
self.History,
debug=self._debug)
except MemoryError:
print '---Data too large for machine memory---'
print 'Tip: use ax or tx during class initialisation'
print '--- to use partial data'
raise
elif filename.endswith('.mat'):
print "---Functionality not yet implemented---"
sys.exit()
else:
print "---Wrong file format---"
sys.exit()
def test_uint16(self):
"""Load an uint16."""
dataset = open_url("http://localhost:8001/", self.app)
self.assertEqual(dataset.types.ui16.dtype, np.dtype(">u2"))
def test_original(self):
"""Test an unmodified call, without function calls."""
original = open_url('/', application=self.app)
self.assertEqual(original.SimpleGrid.SimpleGrid.shape, (2, 3))
def near(x,x0):
"""
Find the index where x has the closer value to x0
"""
dx = x - x0
dx = np.abs(dx)
fn = np.where( dx == dx.min() )
fn = fn[0][0]
return fn
###########################################################################
print 'Accessing dods server'
dataset = open_url('http://tds.hycom.org/thredds/dodsC/GLBa0.08/expt_60.5')
print 'Downloading lon, lat, depth, time arrays'
tt = dataset.MT[:]
depth = dataset['Depth'][:]
lon = dataset.Longitude.array[:]; lon = lon-360
lat = dataset.Latitude.array[:]
# obs: hycom dates are referenced to year 1900
tstart = dt.datetime(2003-1900, 1, 1); tstart = date2num(tstart)
tend = dt.datetime(2006-1900, 1, 1); tend = date2num(tend)
fk = near(depth, 60)
fj = np.where( (lon[1,:] >= -60) & (lon[1,:] <= 30) )
fi = np.where( (lat[:,1] >= -50) & (lat[:,1] <= -5) )
fj1 = fj[0][0]; fj2 = fj[0][1]
fi1 = fi[0][0]; fi2 = fi[0][1]
def test_client(self):
dataset = open_url("http://localhost:8001/", application=self.app)
self.assertEqual(dataset["a.b"].name, "a%2Eb")
self.assertEqual(dataset["a.b"][0], 1)
def DownloadPoints(variable,
latitudes_s,
longitudes_s,
url,
outfile,
session=None):
""" Download time-series for a list of lats/longs from a single opendap dataset (ie CORDEX netcdf).
The output nc file has primary dimensions location and time.
Inputs:
-------
variable - string. variable to download, eg 'tas' or 'pr'
latitudes_s, longitudes_s are python lists
url - opendap url
outfile - string. where to write the file.
session - Note or pydap.cas.esgf session. None for a new session.
Might save a little time if you reuse sessions rather than creating new
ones for each download? Might be more trouble than it is worth, seems to
get time-outs with re-using sesions. Best to just leave as None. But if
you want to try:
session = setup_session(config.openid, config.password, verify=True, check_url=url)
"""
if session == None:
session = setup_session(config.openid,
config.password,
verify=True,
check_url=url)
d = open_url(url, session=session)
# lat/long to grid mapping
Message('download from %s' % url, 2)
####
Message(' ... reading header and extracting pixel indices...',
2,
newline=False)
lat = d['lat'].array.data[:, :]
lon = d['lon'].array.data[:, :]
for i, l in enumerate(longitudes_s):
if l > 180:
longitudes_s[i] = l - 360
lon_shift = lon > 180
lon[lon_shift] = lon[lon_shift] - 360
ns = len(longitudes_s)
rlat_idx = [None] * ns
rlon_idx = [None] * ns
lat_of_pixel = np.zeros(ns, dtype=lat.dtype)
lon_of_pixel = np.zeros(ns, dtype=lat.dtype)
Message('done.', 2)
####
Message(' ... reading and processing header for %d locations...' % ns, 2,
False)
for i in range(ns):
longitude = longitudes_s[i]
latitude = latitudes_s[i]
dist = ((abs(lon - longitude)**2) + (abs(lat - latitude)**2))
mask = dist == np.amin(dist)
lat_of_pixel[i] = lat[mask]
lon_of_pixel[i] = lon[mask]
rlat_idx[i] = np.asscalar(mask.nonzero()[0])
rlon_idx[i] = np.asscalar(mask.nonzero()[1])
Message('done.', 2)
####
# open a new netCDF file for writing, and download data and write it in directly!
Message(' ... downloading and writing to %s...' % outfile, 2)
with Dataset(outfile, 'w', format='NETCDF4') as ncfile:
for name in d.attributes['NC_GLOBAL']:
attr_value = d.attributes['NC_GLOBAL'][name]
if name[0] != '_' and isinstance(attr_value, str):
ncfile.setncattr(name, attr_value)
# create the x and y dimensions.
ncfile.createDimension('location', ns)
ncfile.createDimension('time', None)
times = ncfile.createVariable('time', d['time'].dtype.name, ('time', ))
latitudes = ncfile.createVariable('lat', lat_of_pixel.dtype.name,
('location', ))
longitudes = ncfile.createVariable('lon', lon_of_pixel.dtype.name,
('location', ))
M = ncfile.createVariable(variable, d[variable].array.dtype.name, (
'time',
'location',
))
latitudes[:] = lat_of_pixel
longitudes[:] = lon_of_pixel
times[:] = d['time'].data[:]
for i in range(ns):
Message(' ... station %d of %d' % (i, ns), 2)
M[:, i] = d[variable].array.data[:, rlat_idx[i], rlon_idx[i]]
for cvar in (
'time',
variable,
):
for name in d[cvar].attributes:
attr_value = d[cvar].attributes[name]
if name[0] != '_' and isinstance(attr_value, str):
ncfile.variables[cvar].setncattr(name, attr_value)
Message('done.', 2)
def mapdap(
varname = 'hr24_prcp',
bbox = '-180,-90,180,90',
url = 'http://opendap.bom.gov.au:8080/thredds/dodsC/PASAP/atmos_latest.nc',
timeindex = 'Default',
imgwidth = 256,
imgheight = 256,
request = 'GetMap',
time = 'Default',
save_local_img = False,
colorrange = (-4,4),
palette = 'RdYlGn',
colorbounds = 'Default',
style = 'grid',
ncolors = 10,
mask = -999,
plot_mask = True,
mask_varname = 'mask',
mask_value = 1.0
):
""" Using Basemap, create a contour plot using some dap available data
Data is assumed to have dimensions [time,lat,lon]
TODO -- deal with other shapes
TODO -- determine the dimension ordering using CF convention
varname -- name of variable in opendap file
bbox -- lonmin,latmin,lonmax,latmax for plot
url -- OPEnDAP url
timeindex -- time index to plot
imgwidth,imgheight -- size of png image to return
request -- 'GetMap','GetLegend','GetFullFigure'
time -- time vale to plot. Assumes a particular format."%Y-%m-%dT%H:%M:%S"
mask -- mask out these values
if plot_mask is True, mask_varname and mask_value must be given
"""
transparent = True
lonmin,latmin,lonmax,latmax = tuple([float(a) for a in bbox.rsplit(',')])
# It's not clear there is any point in this. Pydap doesn't actually
# download data until you subscript
if url not in cache:
dset = open_url(url)
else:
dset = cache[url]
# Get the correct time.
time_var = dset['time']
time_units = time_var.attributes['units']
available_times = np.array(time_var[:])
# TODO there is a potential conflict here between time and timeindex.
# On the one hand we want to allow using the actual time value.
# On the other hand we want to make it easy to get a time index
# without knowing the value.
timestep=0
if timeindex == 'Default':
timestep=0
else:
timestep=int(timeindex)
if time != 'Default':
dtime = datetime.datetime.strptime(time, "%Y-%m-%dT%H:%M:%S" )
reftime = date2num(dtime,time_units)
timestep = np.where(available_times >= reftime)[0].min()
# TODO Get only the section of the field we need to plot
# TODO Determine lat/lon box indices and only download this slice
# TODO Set default range (the below does not work)
#colorrange = np.min(var),np.max(var)
lat = dset['lat'][:]
lon = dset['lon'][:]
# CHANGED
var = dset[varname][timestep,:,:]
#xcoords = lonmin,lonmax
#xcoords,lon,var = transform_lons(xcoords,lon,var)
# TODO
# Needs mre thought - the idea here is to only grab a slice of the data
# Need to grab a slightly larger slice of data so that tiling works.
#lat_idx = (lat > latmin) & (lat < latmax)
#lon_idx = (lon > lonmin) & (lon < lonmax)
#lat = dset['lat'][lat_idx]
#lon = dset['lon'][lon_idx]
#latdx1 = np.where(lat_idx)[0].min()
#latdx2 = np.where(lat_idx)[0].max()
#londx1 = np.where(lon_idx)[0].min()
#londx2 = np.where(lon_idx)[0].max()
#var = var[latdx1:latdx2+1,londx1:londx2+1]
#var = dset[varname][timestep,latdx1:latdx2+1,londx1:londx2+1]
# todo clean up this logic
if 'mask' in dset.keys():
if plot_mask:
maskvar = dset['mask'][timestep,:,:]
#maskvar = dset['mask'][timestep,latdx1:latdx2+1,londx1:londx2+1]
varm = np.ma.masked_array(var,mask=maskvar)
mask = varm.mask
else:
varm = np.ma.masked_array(var,mask=np.isinf(var))
xcoords = lonmin,lonmax
# Call the trans_coords function to ensure that basemap is asked to
# plot something sensible.
xcoords,lon,varm = transform_lons(xcoords,lon,varm)
lonmin,lonmax = xcoords
varnc = dset[varname]
try:
var_units = varnc.attributes['units']
except KeyError:
var_units = ''
# Plot the data
# For the basemap drawing we can't go outside the range of coordinates
# WMS requires us to give an empty (transparent) image for these spurious lats
# uc = upper corner, lc = lower corner
bmapuclon=lonmax
bmaplclon=lonmin
bmapuclat=min(90,latmax)
bmaplclat=max(-90,latmin)
if bmaplclat==90:
bmaplclat = 89.0
if bmapuclat==-90:
bmapuclat = -89.0
# TODO set figsize etc here
fig = mpl.figure.Figure()
canvas = FigureCanvas(fig)
ax = fig.add_axes((0,0,1,1),frameon=False,axisbg='k',alpha=0,visible=False)
m = Basemap(projection='cyl',resolution='c',urcrnrlon=bmapuclon,
urcrnrlat=bmapuclat,llcrnrlon=bmaplclon,llcrnrlat=bmaplclat,
suppress_ticks=True,fix_aspect=False,ax=ax)
DPI=100.0
# Convert the latitude extents to Basemap coordinates
bmaplatmin,bmaplonmin = m(latmin,lonmin)
bmaplatmax,bmaplonmax = m(latmax,lonmax)
lon_offset1 = abs(bmaplclon - bmaplonmin)
lat_offset1 = abs(bmaplclat - bmaplatmin)
lon_offset2 = abs(bmapuclon - bmaplonmax)
lat_offset2 = abs(bmapuclat - bmaplatmax)
lon_normstart = lon_offset1 / abs(bmaplonmax - bmaplonmin)
lat_normstart = lat_offset1 / abs(bmaplatmax - bmaplatmin)
ax_xfrac = abs(bmapuclon - bmaplclon)/abs(bmaplonmax - bmaplonmin)
ax_yfrac = abs(bmapuclat - bmaplclat)/abs(bmaplatmax - bmaplatmin)
# Set plot_coords, the plot boundaries. If this is a regular WMS request,
# the plot must fill the figure, with whitespace for invalid regions.
# If it's a full figure, we need to make sure there is space for the legend
# and also for the text.
if request == 'GetFullFigure':
coords = lonmin,latmin,lonmax,latmax
plot_coords = figurePlotDims(imgheight,imgwidth,coords)
else:
plot_coords = (lon_normstart,lat_normstart,ax_xfrac,ax_yfrac)
m = Basemap(projection='cyl',resolution='c',urcrnrlon=bmapuclon,
urcrnrlat=bmapuclat,llcrnrlon=bmaplclon,llcrnrlat=bmaplclat,
suppress_ticks=True,fix_aspect=False,ax=ax)
ax = fig.add_axes(plot_coords,frameon=False,axisbg='k')
m.ax = ax
varm,lonwrap = addcyclic(varm,lon)
x,y = m(*np.meshgrid(lonwrap[:],lat[:]))
""" To plot custom colors
rgb_cmap = mpl.colors.ListedColormap([
(0.0,0.0,0.0),
(0.25,0.25,0.25),
(0.3,0.25,0.25),
(0.5,0.5,0.5),
(0.6,0.5,0.5),
(0.75,0.75,0.75),
(0.75,0.85,0.75),
(1.0,1.0,1.0) ],name='rgbcm')
default_color_bounds = [-1,-0.75,-0.5,-0.25,0.0,0.25,0.5,0.75,1.0]
default_norm = mpl.colors.BoundaryNorm(default_color_bounds, rgb_cmap.N)
m.contourf(x,y,var,cmap=rgb_cmap,norm=default_norm)
contours = m.contour(x,y,var,cmap=rgb_cmap,norm=default_norm)
contours.clabel(colors='k')
"""
colormap = mpl.cm.get_cmap(palette)
# colormap = cmap_discretize(colormap,ncolors)
# if colorbounds = 'Default':
# colorbounds = list(np.arange(colorrange[0],colorrange[1]+increment,increment))
# else:
# colorbounds = list(np.arange(colorrange[0],colorrange[1]+increment,increment))
# Do some checks on the size of the list, and fix if we can
# pass
if style == 'contour':
# Interpolate to a finer resolution
# TODO: make this sensitive to the chosen domain
increment = float(colorrange[1]-colorrange[0]) / float(ncolors-2)
colorbounds = list(np.arange(colorrange[0],colorrange[1]+increment,increment))
# CHANGED
colormap = cmap_discretize(colormap,ncolors)
colvs =[-999]+colorbounds+[999]
lat_idx = np.argsort(lat)
lat = lat[lat_idx]
varm = varm[lat_idx,:]
data_lonmin = min(lonwrap)
data_lonmax = max(lonwrap)
data_latmin = min(lat)
data_latmax = max(lat)
new_lons = np.arange(data_lonmin-1.0,data_lonmax+1.0,1.0)
new_lats = np.arange(data_latmin-1.0,data_latmax+1.0,1.0)
newx,newy = m(*np.meshgrid(new_lons[:],new_lats[:]))
x = newx
y = newy
# Two pass interpolation to deal with the mask.
# The first pass does a bilinear, the next pass does a nearest neighbour to keep the mask
# These steps slow down the plotting significantly
# It's not clear this is working, and the problem is likely solved by
# ensuring the right mask is used!
varm_bl = interp(varm, lonwrap[:], lat[:], newx, newy,order=1)
varm_nn = interp(varm, lonwrap[:], lat[:], newx, newy,order=0)
varm = varm_bl
varm[varm_nn.mask == 1] = varm_nn[varm_nn.mask == 1]
# contourf has an extent keyword (x0,x1,y0,y1)
# return "mapdap\n"
# STUCK it gets stuck here (in apache)
main_render = m.contourf(x,y,varm[:,:],colorbounds,extend='both',cmap=colormap,ax=ax)
contours = m.contour(x,y,varm,colorbounds,colors='k',ax=ax)
contours.clabel(colors='k',rightside_up=True,fmt='%1.1f',inline=True)
elif style == 'grid':
main_render = m.pcolormesh(x,y,varm[:,:],vmin=colorrange[0],vmax=colorrange[1],
cmap=colormap,ax=ax)
elif style == 'grid_threshold':
increment = float(colorrange[1]-colorrange[0]) / float(ncolors)
colorbounds = list(np.arange(colorrange[0],colorrange[1]+increment,increment))
colornorm = mpl.colors.BoundaryNorm(colorbounds,colormap.N)
main_render = m.pcolor(x,y,varm[:,:],vmin=colorrange[0],vmax=colorrange[1],
cmap=colormap,ax=ax,norm=colornorm)
else:
main_render = m.pcolormesh(x,y,varm[:,:],vmin=colorrange[0],vmax=colorrange[1],
cmap=colormap,ax=ax)
fig.set_dpi(DPI)
fig.set_size_inches(imgwidth/DPI,imgheight/DPI)
title_font_size = 9
tick_font_size = 8
if request == 'GetFullFigure':
# Default - draw 5 meridians and 5 parallels
n_merid = 5
n_para = 5
# base depends on zoom
mint = (lonmax - lonmin)/float(n_merid)
base = mint
meridians = [lonmin + i*mint for i in range(n_merid)]
meridians = [ int(base * round( merid / base)) for merid in meridians]
# Some sensible defaults for debugging
#meridians = [45,90,135,180,-135,-90,-45]
pint = int((latmax - latmin)/float(n_para))
base = pint
parallels = [latmin + i*pint for i in range(1,n_para+1)]
parallels = [ int(base * round( para / base)) for para in parallels]
#parallels = [-60,-40,-20,0,20,40,60]
#parallels = [((parallel + 180.) % 360.) - 180. for parallel in parallels]
m.drawcoastlines(ax=ax)
m.drawmeridians(meridians,labels=[0,1,0,1],fmt='%3.1f',fontsize=tick_font_size)
m.drawparallels(parallels,labels=[1,0,0,0],fmt='%3.1f',fontsize=tick_font_size)
m.drawparallels([0],linewidth=1,dashes=[1,0],labels=[0,1,1,1],fontsize=tick_font_size)
titlex,titley = (0.05,0.98)
# CHANGED
# STUCK getting an error somewhere in this function
# title = get_pasap_plot_title(dset,varname=varname,timestep=timestep)
title = "We're getting errors in the get title function"
fig.text(titlex,titley,title,va='top',fontsize=title_font_size)
colorbar_font_size = 8
if request == 'GetLegendGraphic':
# Currently we make the plot, and then if the legend is asked for
# we use the plot as the basis for the legend. This is not optimal.
# Instead we should be making the legend manually. However we need
# to set up more variables, and ensure there is a sensible min and max.
# See the plot_custom_colors code above
fig = mpl.figure.Figure(figsize=(64/DPI,256/DPI))
canvas = FigureCanvas(fig)
# make some axes
cax = fig.add_axes([0,0.1,0.2,0.8],axisbg='k')
# put a legend in the axes
cbar = fig.colorbar(main_render,cax=cax,extend='both',format='%1.1f')
cbar.set_label(var_units,fontsize=colorbar_font_size)
for t in cbar.ax.get_yticklabels():
t.set_fontsize(colorbar_font_size)
# i.e. you don't need to plot the figure...
#fig.colorbar(filled_contours,cax=cax,norm=colornorm,boundaries=colvs,values=colvs,
# ticks=colorbounds,spacing='proportional')
elif request == 'GetFullFigure':
# Add the legend to the figure itself.
# Figure layout parameters
# plot_coords = tuple with (xi,yi,dx,dy)
# legend_coords = tuple with (xi,yi,dx,dy) as per mpl convention
# First change the plot coordinates so that they do not cover the whole image
legend_coords = (0.8,0.1,0.02,plot_coords[3])
cax = fig.add_axes(legend_coords,axisbg='k')
cbar = fig.colorbar(main_render,cax=cax,extend='both')
for t in cbar.ax.get_yticklabels():
t.set_fontsize(colorbar_font_size)
cbar.set_label(var_units,fontsize=colorbar_font_size)
transparent=False
# Experimenting here with custom color map and ticks. Assigning everything manually
# (e.g. ticks=[-2,-1,0,1,2]) is easy. Doing it in an automated way given a range is
# hard...
#fig.colorbar(filled_contours,cax=cax,boundaries=colvs,ticks=colorbounds)
#,norm=colornorm,#boundaries=colvs,values=colvs, #extend='both')
imgdata = StringIO.StringIO()
fig.savefig(imgdata,format='png',transparent=transparent)
if save_local_img:
fig.savefig('map_plot_wms_output.png',format='png')
return
if url not in cache:
cache[url] = dset
value = imgdata.getvalue()
#imgdata.close()
fig = None
return value
def basemap_detail(lat,lon,bathy,draw_parallels,*parallels_interval):
## plot the coastline
url='http://geoport.whoi.edu/thredds/dodsC/bathy/gom03_v03'
def get_index_latlon(url):# use the function to calculate the minlat,minlon,maxlat,maxlon location
try:
dataset=open_url(url)
except:
print "please check your url!"
sys.exit(0)
basemap_lat=dataset['lat']
basemap_lon=dataset['lon']
basemap_topo=dataset['topo']
# add the detail of basemap
minlat=min(lat)-0.01
maxlat=max(lat)+0.01
minlon=min(lon)+0.01
maxlon=max(lon)-0.01
index_minlat=int(round(np.interp(minlat,basemap_lat,range(0,basemap_lat.shape[0]))))
index_maxlat=int(round(np.interp(maxlat,basemap_lat,range(0,basemap_lat.shape[0]))))
index_minlon=int(round(np.interp(minlon,basemap_lon,range(0,basemap_lon.shape[0]))))
index_maxlon=int(round(np.interp(maxlon,basemap_lon,range(0,basemap_lon.shape[0]))))
return index_minlat,index_maxlat,index_minlon,index_maxlon,basemap_lat,basemap_lon,basemap_topo
index_minlat,index_maxlat,index_minlon,index_maxlon,basemap_lat,basemap_lon,basemap_topo = get_index_latlon(url)
#print index_minlat,index_maxlat,index_minlon,index_maxlon
if index_minlat==0 or index_maxlat==0 or index_minlon==0 or index_maxlon==0:
url='http://geoport.whoi.edu/thredds/dodsC/bathy/crm_vol1.nc'
try:
dataset=open_url(url)
except:
print "please check your url!"
sys.exit(0)
basemap_lat=dataset['lat']
basemap_lon=dataset['lon']
basemap_topo=dataset['topo']
# add the detail of basemap
minlat=min(lat)-0.01
maxlat=max(lat)+0.01
minlon=min(lon)+0.01
maxlon=max(lon)-0.01
basemap_lat=[float(i) for i in basemap_lat]
basemap_lat.reverse()
range_basemap_lat=range(len(basemap_lat))
range_basemap_lat.reverse()
index_minlat=int(round(np.interp(minlat,basemap_lat,range_basemap_lat)))
index_maxlat=int(round(np.interp(maxlat,basemap_lat,range_basemap_lat)))
index_minlon=int(round(np.interp(minlon,basemap_lon,range(0,basemap_lon.shape[0]))))
index_maxlon=int(round(np.interp(maxlon,basemap_lon,range(0,basemap_lon.shape[0]))))
min_index_lat=min(index_minlat,index_maxlat)
max_index_lat=max(index_minlat,index_maxlat)
X,Y=np.meshgrid(basemap_lon[index_minlon-15:index_maxlon+15],basemap_lat[min_index_lat-15:max_index_lat+15])
# You can set negative contours to be solid instead of dashed:
matplotlib.rcParams['contour.negative_linestyle'] = 'solid'
#plot the bathy
if bathy==True:
CS=plt.contour(X,Y,basemap_topo.topo[min_index_lat-15:max_index_lat+15,index_minlon-15:index_maxlon+15],3,colors='gray',linewidths=0.1)
plt.clabel(CS, fontsize=7,fmt='%5.0f', inline=1)
#plt.clabel(cs, fontsize=9, inline=1,fmt='%5.0f'+"m")
plt.contourf(X,Y,basemap_topo.topo[min_index_lat-15:max_index_lat+15,index_minlon-15:index_maxlon+15],[0,1000],colors='grey')
ax=plt.gca()
ax.set_xticklabels([])
ax.set_yticklabels([])
#set up the map in a Equidistant Cylindrical projection
#m.drawmapboundary()
#draw major rivers
#m.drawrivers()
if draw_parallels==True:
from mpl_toolkits.basemap import Basemap
m = Basemap(projection='cyl',llcrnrlat=min(lat),urcrnrlat=max(lat),\
llcrnrlon=min(lon),urcrnrlon=max(lon),resolution='h',suppress_ticks=False)#,fix_aspect=False)
if len(parallels_interval)<1:
parallels_interval=1
#draw parallels
m.drawparallels(np.arange(int(min(lat)),int(max(lat)),float(parallels_interval)),labels=[1,0,0,0],fmt=lat2str,dashes=[2,2])
#draw meridians
m.drawmeridians(np.arange(int(min(lon)),int(max(lon)),float(parallels_interval)),labels=[0,0,0,1],fmt=lon2str,dashes=[2,2])
else:
parallels_interval=parallels_interval[0]
#draw parallels
m.drawparallels(np.arange(round(min(lat),3),round(max(lat),3),parallels_interval),labels=[1,0,0,0],fmt=lat2str,dashes=[2,2])
#draw meridians
m.drawmeridians(np.arange(round(min(lon),3),round(max(lon),3),parallels_interval),labels=[0,0,0,1],fmt=lon2str,dashes=[2,2])
'''
Script para baixar os resultados de vento a 10 metros do GFS
via opendap. As informações são salvas em um arquivo netCDF.
Criado por Rafael Vieira - maio/2016
'''
from pydap.client import open_url
import numpy as np
from netCDF4 import Dataset
#acessar os resultados via opendap no servidor
u_component = open_url(
'http://nomads.ncep.noaa.gov:80/dods/gfs_0p25_1hr/gfs20160616/gfs_0p25_1hr_00z'
)['ugrd10m']
v_component = open_url(
'http://nomads.ncep.noaa.gov:80/dods/gfs_0p25_1hr/gfs20160616/gfs_0p25_1hr_00z'
)['vgrd10m']
tempo = open_url(
'http://nomads.ncep.noaa.gov:80/dods/gfs_0p25_1hr/gfs20160616/gfs_0p25_1hr_00z'
)['time'][0:121:3]
lats = open_url(
'http://nomads.ncep.noaa.gov:80/dods/gfs_0p25_1hr/gfs20160616/gfs_0p25_1hr_00z'
)['lat'][:]
lons = open_url(
'http://nomads.ncep.noaa.gov:80/dods/gfs_0p25_1hr/gfs20160616/gfs_0p25_1hr_00z'
)['lon'][:]
ugrd_10m = u_component.array[0:121:3, :, :]
vgrd_10m = v_component.array[0:121:3, :, :]
lonRange = '[' + str(lonRangeMin) + ':1:' + str(lonRangeMax) + ']'
latRange = '[' + str(latRangeMin) + ':1:' + str(latRangeMax) + ']'
timeRange = '[' + str(timeRangeMin) + ':1:' + str((timeRangeMax)) + ']' #+3
uwndRange = timeRange + latRange + lonRange
vwndRange = timeRange + latRange + lonRange
generalUrl = 'https://thredds.jpl.nasa.gov/thredds/dodsC/ncml_aggregation/OceanWinds/ccmp/aggregate__CCMP_MEASURES_ATLAS_L4_OW_L3_0_WIND_VECTORS_FLK.ncml'
resultUrl = generalUrl + '?' + 'lon' + lonRange + ',' + 'lat' + latRange + ',' + 'time' + timeRange + ',' + 'uwnd' + uwndRange + ',' + 'vwnd' + vwndRange
print('')
print('URL:', resultUrl)
# OPEN URL
dataset = open_url(resultUrl)
# DOWNLOAD DATA BASE TYPE
lonData = dataset.lon.data[lonArrayMin:lonArrayMax:precisionLon]
latData = dataset.lat.data[latArrayMin:latArrayMax:precisionLat]
print('USED VALUES LON: ', lonData)
print('USED VALUES LAT: ', latData)
timeData = dataset.time.data[timeArray]
passedHours = timeData[0]
usedDate = startDate + datetime.timedelta(hours=passedHours)
print('USED DATE: ', usedDate)
## 12: only "00" and "12" hour tick labels
## 24: no hour tick labels
hr_tick_fq = 12
start = datetime.fromtimestamp(time.mktime(time.strptime(st, "%Y%m%d%H")))
end = datetime.fromtimestamp(time.mktime(time.strptime(et, "%Y%m%d%H")))
times = []
levs = []
bs_data = []
for timestamp in datespan(start, end, delta=timedelta(days=1)):
src = "http://asn.mesowest.net/data/opendap/PUBLIC/{0:%Y/%m/%d/}{1}_{0:%Y%m%d}.h5"
source = src.format(timestamp, station)
print source
## pydap created a "dataset" object, which we can see has all the structure of the file. There is a HEIGHT attribute, which contains the height dimension, and a data attribute which contains all the different data columns.
data = pd.open_url(source)
height = data['HEIGHT'][:]
tt = data['data']['DATTIM'][:]
bs = data['data']['BS'][:]
#print height.shape, tt.shape, bs.shape
if len(times) == 0:
times = tt
levs = height
bs_data = bs
else:
if (levs != height).sum() == 0:
times = np.append(times, tt)
#levs = np.append(levs,height)
bs_data = np.append(bs_data, bs, 0)
else:
print "attention: different levels"
def canhandle(url):
try:
return isinstance(open_url(url), dap.model.DatasetType)
except Exception:
return False
# University of Alabama in Huntsville
#
#
########################################################
# In[ ]:
import pydap
from pydap.client import open_url
# # Open data stream using OPeNDAP link to file and look for data fields
# In[ ]:
datafile = open_url(
'https://ghrc.nsstc.nasa.gov/opendap/fieldCampaigns/hs3/HAMSR/data/2013/HAMSR_L2_20130915T061329_20130916T050512_v01.nc'
)
print datafile.keys()
# # Let's plot the ham_dBz field
# In[ ]:
#Import needed Python packages
import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.basemap import Basemap
from scipy import ndimage
from datetime import datetime
# # Define the start and end date/time along the flight track
f = open('timereport.csv', 'w')
#g=open('timereportsite.csv','w')
'''
urllatlon = 'http://gisweb.wh.whoi.edu:8080/dods/whoi/emolt_site?emolt_site.SITE'
dataset = open_url(urllatlon)
print dataset
var = dataset['emolt_site']
SITE = list(var.SITE)
print "SITE list has been generated"
#for i in range(len([0,1])):
'''
SITE=['AB01','AG01','BA01','BA02','BC01','BD01','BF01','BI02','BI01','BM01','BM02','BN01','BS02','CJ01','CP01','DC01','DJ01','DK01','DMF1','ET01','GS01','JA01','JC01','JS06','JT04','KO01','MF02','MM01','MW01','NL01','PF01','PM02','PM03','PW01','RA01','RM02','RM04','SJ01','TA14','TA15','TS01']
for i in range(len(SITE)):
print SITE[i]
url='http://gisweb.wh.whoi.edu:8080/dods/whoi/emolt_sensor?emolt_sensor.TIME_LOCAL&emolt_sensor.SITE='
dataset=open_url(url+'"'+SITE[i]+'"')
var=dataset['emolt_sensor']
print 'hold on ... extracting your eMOLT mooring data'
year_month_day = list(var.TIME_LOCAL)
timelocal=[]
for j in range(len(year_month_day)):
timelocal.append(datetime.strptime(year_month_day[j],"%Y-%m-%d"))
index = range(len(timelocal))
index.sort(lambda x, y:cmp(timelocal[x], timelocal[y]))
timelocal = [timelocal[ii] for ii in index]
print 'now generating a datetime'
timepd=pd.DataFrame(range(len(timelocal)),index=timelocal)
timepd['Year']=timepd.index.year
year=unique(timepd['Year'])
monthall=[]
if len(year)>=minyear:
def _dap_open(self, url):
pydap.lib.PROXY = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP,
'localhost', PROXY_PORT)
dataset = open_url(url)
return dataset
#elif ((year >= 1993) & (year <= 2000)):
# file_tag = 'MERRA201'
#elif ((year >= 2001) & (year <= 2009)):
# file_tag = 'MERRA301'
#elif (year >= 2010):
# file_tag = 'MERRA300'
file_tag = 'MERRA2_400'
#read grid and variable attributes from the first file
year_tag = '%04d' % year
month_tag = '01'
day_tag = '01'
date_tag = year_tag + month_tag + day_tag
url = server + '/opendap/MERRA2/M2T1NXRAD.5.12.4/' + year_tag + '/' + month_tag + '/' + \
file_tag + '.tavg1_2d_rad_Nx.' + date_tag + '.nc4'
dataset = open_url(url)
lon = dataset['lon'][:]
#shift data between 0 and 360 deg.
gidx = np.where(np.abs(lon) < 1.0e-10)[0][0]
lon = lon + 180
lat = dataset['lat'][:]
spval = dataset[invarname].missing_value
units = dataset[invarname].units
long_name = dataset[invarname].long_name
#get data from NASA opendap
for month in range(12):
nday = 0
#create ROMS forcing file
month_tag = '%02d' % (month + 1)
def load_ASN_data(st, et, bottom, top, station):
## Loads the data from the ASN network through OPeNDAP
print 'loading data...'
#___________________Load Data__________________________
start = datetime.strptime(st, "%Y%m%d%H")
end = datetime.strptime(et, "%Y%m%d%H")
times = []
levs = []
bs_data = []
fileList = []
src = "http://asn.mesowest.net/data/opendap/PUBLIC/{0:%Y/%m/%d/}{1}_{0:%Y%m%d}.h5"
# Create file_list from start time and end time
if st[0:8] != et[0:8]:
for timestamp in datespan(start, end, delta=timedelta(hours=1)):
source = src.format(timestamp, station)
if source in fileList:
pass
else:
fileList.append(source)
else:
timestamp = start
source = src.format(timestamp, station)
fileList.append(source)
#print fileList
# Get data from each file in file_list using pydap
for ff in fileList:
print ff
## pydap created a "dataset" object, which we can see has all the structure of the file. There is a HEIGHT attribute, which contains the height dimension, and a data attribute which contains all the different data columns.
data = pd.open_url(ff)
height = data['HEIGHT'][:]
tt = data['data']['DATTIM'][:]
bs = data['data']['BS'][:]
#print height.shape, tt.shape, bs.shape
if len(times) == 0:
times = tt
levs = height
bs_data = bs
else:
if (levs != height).sum() == 0:
times = np.append(times, tt)
#levs = np.append(levs,height)
bs_data = np.append(bs_data, bs, 0)
else:
print "attention: different levels"
pass
# Convert times into datetime array
times_utc = np.array([datetime.utcfromtimestamp(i) for i in times])
times_local = np.array([datetime.fromtimestamp(i) for i in times])
# Get index for desired times and create new times array
#utc
time_start_index_utc = find_nearest(times_utc, start)
time_end_index_utc = find_nearest(times_utc, end)
times_utc = times_utc[time_start_index_utc:time_end_index_utc]
#local
time_start_index_local = find_nearest(times_local, start)
time_end_index_local = find_nearest(times_local, end)
times_local = times_local[time_start_index_local:time_end_index_local]
# Find index for height range requested and create new height array
height_bottom_index = np.where(height == bottom)
height_top_index = np.where(height == top)
height = height[height_bottom_index[0]:height_top_index[0] +
1] # need plus one to include last index
# Slice the backscatter data for desired times and heights
bs_final = bs_data[time_start_index_utc:time_end_index_utc,
height_bottom_index[0]:height_top_index[0] + 1]
bs_final_local = bs_data[time_start_index_local:time_end_index_local,
height_bottom_index[0]:height_top_index[0] + 1]
return bs_final, bs_final_local, times_utc, times_local, height
def test_axis_mean(self):
"""Test the mean over an axis, returning a scalar."""
original = open_url('/', application=self.app)
dataset = original.functions.mean(original.SimpleGrid.x)
self.assertEqual(dataset.x.shape, ())
np.testing.assert_array_equal(dataset.x.data, np.array(1.0))
# Put your desired plot date in here
# YYYY,MM,DD
plot_date = datetime(2017, 4, 17)
print(plot_date)
# In[4]:
# Link OPeNDAP datasets
modis_sst_url = 'http://oceanus.meas.ncsu.edu:8080/thredds/dodsC/secoora/modis/sst.nc'
modis_chla_url = 'http://oceanus.meas.ncsu.edu:8080/thredds/dodsC/secoora/modis/chla.nc'
dineof_sst_url = 'http://oceanus.meas.ncsu.edu:8080/thredds/dodsC/secoora/dineof/sst.nc'
dineof_chla_url = 'http://oceanus.meas.ncsu.edu:8080/thredds/dodsC/secoora/dineof/chla.nc'
modis_sst_dataset = open_url(modis_sst_url,output_grid=False)
modis_chla_dataset = open_url(modis_chla_url,output_grid=False)
dineof_sst_dataset = open_url(dineof_sst_url,output_grid=False)
dineof_chla_dataset = open_url(dineof_chla_url,output_grid=False)
print('Available data:')
print('MODIS SST:', modis_sst_dataset.keys)
print('MODIS Chla:', modis_chla_dataset.keys)
print('DINEOF SST:', dineof_sst_dataset.keys)
print('DINEOF Chla:', dineof_chla_dataset.keys)
# In[5]:
# Find time index from available times
def test_open_url(self):
"""Open an URL and check dataset keys."""
dataset = open_url('http://localhost:8001/', self.app)
self.assertEqual(list(dataset.keys()), ["cast"])
def test_pydap(self):
if not CFG.get_safe('bootstrap.use_pydap', False):
raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)')
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 10)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
pydap_host = CFG.get_safe('server.pydap.host', 'localhost')
pydap_port = CFG.get_safe('server.pydap.port', 8001)
url = 'http://%s:%s/%s' % (pydap_host, pydap_port, dataset_id)
for i in xrange(
3
): # Do it three times to test that the cache doesn't corrupt the requests/responses
ds = open_url(url)
np.testing.assert_array_equal(ds['time'][:], np.arange(10))
untested = []
for k, v in rdt.iteritems():
if k == rdt.temporal_parameter:
continue
context = rdt.context(k)
if isinstance(context.param_type, QuantityType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, ArrayType):
if context.param_type.inner_encoding is None:
values = np.empty(rdt[k].shape, dtype='O')
for i, obj in enumerate(rdt[k]):
values[i] = str(obj)
np.testing.assert_array_equal(ds[k][k][:][0], values)
elif len(rdt[k].shape) > 1:
values = np.empty(rdt[k].shape[0], dtype='O')
for i in xrange(rdt[k].shape[0]):
values[i] = ','.join(
map(lambda x: str(x), rdt[k][i].tolist()))
elif isinstance(context.param_type, ConstantType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, CategoryType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
else:
untested.append('%s (%s)' % (k, context.param_type))
if untested:
raise AssertionError('Untested parameters: %s' % untested)
