def getCoastline(res='low'):
"""
Downloads NOAA coastline from OCS ftp
"""
coastlineFile = "./coastline.txt"
request = 'ftp://ocsftp.ncd.noaa.gov/estofs/data/'
if res is 'low':
request = request + 'noaa_coastline_world.dat'
else:
request = request + 'noaa_coastline_usa.dat'
transfer.download(request, coastlineFile)
# if not os.path.exists(coastlineFile):
# print '[info]: retrieving coastline from ', request
# urllib.urlretrieve (request, coastlineFile)
f = open(coastlineFile, 'r')
xc = []
yc = []
for line in f:
xc.append(float(line.split()[0]))
yc.append(float(line.split()[1]))
f.close()
return {'lon': xc, 'lat': yc}
def getCities():
"""
"""
citiesFile = "./cities.csv"
request = 'ftp://ocsftp.ncd.noaa.gov/estofs/data/cities.csv'
transfer.download(request, citiesFile)
return readCities(citiesFile)
def bias_surface(inputFile, outputFile, mapFile, toolkitPath, gridFile):
sys.path.insert(0, toolkitPath)
from csdlpy.obs import coops
from csdlpy import transfer
from csdlpy import adcirc
from csdlpy import interp
# Read bias table
xo, yo, vo, datespan = coops.read_bias_table(inputFile)
# Read / download grid
if "ftp://" in gridFile or "http://" in gridFile or "https://" in gridFile:
transfer.download(gridFile, 'fort.14')
gridFile = 'fort.14'
grid = adcirc.readGrid(gridFile)
print '[test]: # Compute bias surface on the grid...'
vg = np.zeros(len(grid['depth']), dtype=float)
# Interpolation parameters...............................................
z_full = 0. # meters
z_zero = 200. # meters
p = 2.0 # scalar
R = 2.0 # degrees
#........................................................................
print '[test]: ## Interpolate on the shelf...'
ind_shelf = np.where(grid['depth'] < z_zero)[0]
vg[ind_shelf] = interp.shepard_idw(xo, yo, vo, grid['lon'][ind_shelf],
grid['lat'][ind_shelf], p)
print '[test]: ## Taper by depth...'
ind_taper = np.where(
np.logical_and(z_full <= grid['depth'], z_zero >= grid['depth']))[0]
vg[ind_taper] = interp.taper_linear(z_full, z_zero,
grid['depth'][ind_taper],
vg[ind_taper])
print '[test]: ## Zero out the results that are too distant from data'
dist = interp.distance_matrix(xo, yo, grid['lon'], grid['lat'])
for n in range(len(grid['lon'])):
if np.min(dist[:, n]) > R:
vg[n] = 0.
print '[test] :## Write out Offset file'
adcirc.writeOffset63(vg, outputFile)
# Plot
surface_map(grid, vg, toolkitPath, datespan, mapFile)
return vg
def getCoastline(res='low'):
"""
Downloads NOAA coastline from OCS ftp
"""
coastlineFile = "./coastline.txt"
request = 'ftp://ocsftp.ncd.noaa.gov/estofs/data/'
if res is 'low':
request = request + 'noaa_coastline_world.dat'
else:
request = request + 'noaa_coastline_usa.dat'
transfer.download(request, coastlineFile)
# if not os.path.exists(coastlineFile):
# print '[info]: retrieving coastline from ', request
# urllib.urlretrieve (request, coastlineFile)
return readCoastline(coastlineFile)
def getCities():
"""
"""
citiesFile = "./cities.csv"
request = 'ftp://ocsftp.ncd.noaa.gov/estofs/data/cities.csv'
transfer.download(request, citiesFile)
import csv
f = open(citiesFile)
s = csv.reader(f)
next(s, None)
xc = []
yc = []
name = []
for row in s:
xc.append(float(row[3]))
yc.append(float(row[2]))
name.append(row[0])
f.close()
return {'lon': xc, 'lat': yc, 'name': name}
def test_csdlpy(toolkitPath):
sys.path.insert(0, toolkitPath)
from csdlpy import version
ver = version.__version__
#........................................................................
# Input
webLoc_gridFile = "ftp://ocsftp.ncd.noaa.gov/estofs/atl/fort.14"
webLoc_stnFile = "ftp://ocsftp.ncd.noaa.gov/estofs/atl/stations.txt"
webLoc_maxeleFile = \
"ftp://ocsftp.ncd.noaa.gov/estofs/hsofs-atl/2016_Matthew/best.2016092300.biasCorr/maxele.63.nc"
webLoc_trackFile = \
"ftp://ocsftp.ncd.noaa.gov/estofs/hsofs-atl/2016_Matthew/best.2016092300/fort.22"
#........................................................................
# Output
tableFile = './bias_table.csv'
offsetFile = './Offset.63'
biasPlotFile = './test_csdlpy_2016_Matthew_bias.png'
maxelePlotFile = './test_csdlpy_2016_Matthew_maxele.png'
#........................................................................
print '\n***************************************************************\n'
now = dt.now() # timer
print '[test]: Initiating csdlpy version ' + ver + ' test at ' + \
dt.strftime(now,'%Y/%m/%d %H:%M:%S')
print '[test]: sourced from ', toolkitPath + '\n'
print '[test]: We will attempt to download these required files: '
print webLoc_gridFile
print webLoc_stnFile
print webLoc_maxeleFile
print webLoc_trackFile
print '[test]: Otherwise, you need to provide them to your workdir manually.\n'
print '[warn]: If you do not have web access, comment out CO-OPS data block.\n'
print '[test]: Test will produce the following files:'
print tableFile
print offsetFile
print biasPlotFile
print maxelePlotFile
print '\n'
print '[test]: Please share your benchmark result and log file with us'
print '[test]: [email protected]\n'
from csdlpy import adcirc
from csdlpy import transfer
from csdlpy import plotter
from csdlpy import obs
from csdlpy import interp
from csdlpy import atcf
print '[test]: # Download ESTOFS-Atl grid file...'
gridFile = "./fort.14"
transfer.download(webLoc_gridFile, gridFile)
grid = adcirc.readGrid(gridFile)
# CO-OPS data block starts ###############################################
print '[test]: # Compute data biases...'
if not os.path.exists(tableFile):
obs.coops.bias_table(tableFile, 10.0, now=dt(2016, 9, 23))
xo, yo, vo = obs.coops.read_bias_table(tableFile)
print '[test]: # Compute bias surface on the grid...'
vg = np.zeros(len(grid['depth']), dtype=float)
# Interpolation parameters...............................................
z_full = 0. # meters
z_zero = 200. # meters
p = 2.0 # scalar
R = 2.0 # degrees
#........................................................................
print '[test]: ## Interpolate on the shelf...'
ind_shelf = np.where(grid['depth'] < z_zero)[0]
vg[ind_shelf] = interp.shepard_idw(xo, yo, vo, grid['lon'][ind_shelf],
grid['lat'][ind_shelf], p)
print '[test]: ## Taper by depth...'
ind_taper = np.where(
np.logical_and(z_full <= grid['depth'], z_zero >= grid['depth']))[0]
vg[ind_taper] = interp.taper_linear(z_full, z_zero,
grid['depth'][ind_taper],
vg[ind_taper])
print '[test]: ## Zero out the results that are too distant from data'
dist = interp.distance_matrix(xo, yo, grid['lon'], grid['lat'])
for n in range(len(grid['lon'])):
if np.min(dist[:, n]) > R:
vg[n] = 0.
print '[test] :## Write out Offset file'
adcirc.writeOffset63(vg, offsetFile)
print '[test]: ## Plot bias surface and data'
plotter.plotMap(grid['lon'], grid['lat'], fig_w=8.0)
plotter.addSurface(grid, vg, clim=[-0.3, 0.3])
plotter.addTriangles((xo, yo, vo), clim=[-0.3, 0.3])
plt.title(grid['GridDescription'], fontsize=8)
plt.savefig(biasPlotFile)
plt.close()
# CO-OPS data block ends #################################################
print '[test]: # Download stations file'
stnFile = "./stations.txt"
transfer.download(webLoc_stnFile, stnFile)
stations = adcirc.read_adcirc_stations(open(stnFile, 'r'))
print '[test]: # Download 2016 Matthew hindcast maxele'
maxeleFile = "./maxele.63.nc"
transfer.download(webLoc_maxeleFile, maxeleFile)
maxele = adcirc.readSurfaceField(maxeleFile)
print '[test]: # Download 2016 Matthew best track'
trackFile = "./fort.22"
transfer.download(webLoc_trackFile, trackFile)
track = atcf.readTrack(trackFile)
print '[test]: # Plot maxele file '
plotter.plotMap(grid['lon'], grid['lat'], fig_w=8.0)
plotter.addSurface(grid, maxele['value'], clim=[0., 3.])
plt.plot(track['lon'], track['lat'], 'o-k', markersize=2)
plt.title(grid['GridDescription'], fontsize=8)
plt.scatter(stations['lon'],
stations['lat'],
c='r',
marker='o',
s=20,
edgecolors='w')
for n in range(len(stations['lon'])):
plt.text(stations['lon'][n],
stations['lat'][n],
str(n + 1).zfill(3),
fontsize=6)
plt.savefig(maxelePlotFile)
print '\n'
print '[test]:[total runtime]: ', (dt.now() - now).seconds, ' sec'
transfer.cleanup()
def hsofs_plots(params, inputPath, stormID, inputCycle, outputPath,
toolkitPath):
sys.path.insert(0, toolkitPath)
from csdlpy import estofs
from csdlpy import adcirc
from csdlpy import transfer
from csdlpy import plotter
from csdlpy import atcf
from csdlpy.obs import coops
gridFile = 'fort.14'
trkFile = 'trk.dat'
advFile = 'adv.dat'
transfer.download(params['gridPath'], gridFile)
transfer.download(params['bestTrackURL'], trkFile)
transfer.download(params['advTrackURL'], advFile)
grid = adcirc.readGrid(gridFile)
trk = atcf.readTrack(trkFile)
# nhctrk
ens = 'nhctrk'
ncfile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.fields.maxele.nc'
trkFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.surfaceforcing'
pointsFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.points.waterlevel.nc'
maxele = estofs.getFieldsWaterlevel(ncfile, 'zeta_max')
track_nhctrk = atcf.readTrack(trkFile)
points_nhctrk = estofs.getPointsWaterlevel(pointsFile)
# Plot maxeles maxele_nhctrk
f = plotter.plotMap(params['lonlim'], params['latlim'], fig_w=10.)
plotter.addSurface(grid, maxele['value'], clim=params['clim'])
plt.plot(track_nhctrk['lon'],
track_nhctrk['lat'],
'o-k',
markersize=1,
zorder=10)
title = 'HSOFS experimental ' + stormID + '.' + inputCycle + '.' + ens
plt.text (params['lonlim'][0]+0.03, \
params['latlim'][0]+0.03, \
title )
plotter.save(
title, outputPath + '/' + stormID + '.' + inputCycle + '.' + ens +
'.maxele.png')
plt.close(f)
# 2
ens = 'slowerSpeed'
ncfile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.fields.maxele.nc'
trkFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.surfaceforcing'
pointsFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.points.waterlevel.nc'
maxele = estofs.getFieldsWaterlevel(ncfile, 'zeta_max')
track_lowerSpeed = atcf.readTrack(trkFile)
points_lowerSpeed = estofs.getPointsWaterlevel(pointsFile)
# Plot maxeles maxele_nhctrk
f = plotter.plotMap(params['lonlim'], params['latlim'], fig_w=10.)
plotter.addSurface(grid, maxele['value'], clim=params['clim'])
plt.plot(track_lowerSpeed['lon'],
track_lowerSpeed['lat'],
'o-k',
markersize=1,
zorder=10)
title = 'HSOFS experimental ' + stormID + '.' + inputCycle + '.' + ens
plt.text (params['lonlim'][0]+0.03, \
params['latlim'][0]+0.03, \
title )
plotter.save(
title, outputPath + '/' + stormID + '.' + inputCycle + '.' + ens +
'.maxele.png')
plt.close(f)
# 3
ens = 'shiftRight'
ncfile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.fields.maxele.nc'
trkFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.surfaceforcing'
pointsFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.points.waterlevel.nc'
maxele = estofs.getFieldsWaterlevel(ncfile, 'zeta_max')
track_shiftRight = atcf.readTrack(trkFile)
points_shiftRight = estofs.getPointsWaterlevel(pointsFile)
# Plot maxeles maxele_nhctrk
f = plotter.plotMap(params['lonlim'], params['latlim'], fig_w=10.)
plotter.addSurface(grid, maxele['value'], clim=params['clim'])
plt.plot(track_shiftRight['lon'],
track_shiftRight['lat'],
'o-k',
markersize=1,
zorder=10)
title = 'HSOFS experimental ' + stormID + '.' + inputCycle + '.' + ens
plt.text (params['lonlim'][0]+0.03, \
params['latlim'][0]+0.03, \
title )
plotter.save(
title, outputPath + '/' + stormID + '.' + inputCycle + '.' + ens +
'.maxele.png')
plt.close(f)
# 4
ens = 'shiftLeft'
ncfile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.fields.maxele.nc'
trkFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.surfaceforcing'
pointsFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.points.waterlevel.nc'
maxele = estofs.getFieldsWaterlevel(ncfile, 'zeta_max')
track_shiftLeft = atcf.readTrack(trkFile)
points_shiftLeft = estofs.getPointsWaterlevel(pointsFile)
# Plot maxeles maxele_nhctrk
f = plotter.plotMap(params['lonlim'], params['latlim'], fig_w=10.)
plotter.addSurface(grid, maxele['value'], clim=params['clim'])
plt.plot(track_shiftLeft['lon'],
track_shiftLeft['lat'],
'o-k',
markersize=1,
zorder=10)
title = 'HSOFS experimental ' + stormID + '.' + inputCycle + '.' + ens
plt.text (params['lonlim'][0]+0.03, \
params['latlim'][0]+0.03, \
title )
plotter.save(
title, outputPath + '/' + stormID + '.' + inputCycle + '.' + ens +
'.maxele.png')
plt.close(f)
# 5
ens = 'higherSpeed'
ncfile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.fields.maxele.nc'
trkFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.surfaceforcing'
pointsFile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.' + ens + '.points.waterlevel.nc'
maxele = estofs.getFieldsWaterlevel(ncfile, 'zeta_max')
track_higherSpeed = atcf.readTrack(trkFile)
points_higherSpeed = estofs.getPointsWaterlevel(pointsFile)
# Plot maxeles maxele_nhctrk
f = plotter.plotMap(params['lonlim'], params['latlim'], fig_w=10.)
plotter.addSurface(grid, maxele['value'], clim=params['clim'])
plt.plot(track_higherSpeed['lon'],
track_higherSpeed['lat'],
'o-k',
markersize=1,
zorder=10)
title = 'HSOFS experimental ' + stormID + '.' + inputCycle + '.' + ens
plt.text (params['lonlim'][0]+0.03, \
params['latlim'][0]+0.03, \
title )
plotter.save(
title, outputPath + '/' + stormID + '.' + inputCycle + '.' + ens +
'.maxele.png')
plt.close(f)
# MaxPS
maxPSfile = inputPath + '/hsofs.' + stormID + '.' + inputCycle + '.fields.maxPS.nc'
maxPS = estofs.getFieldsWaterlevel(maxPSfile, 'zeta_max')
# Plot maxeles maxPS
f = plotter.plotMap(params['lonlim'], params['latlim'], fig_w=10.)
plotter.addSurface(grid, maxPS['value'], clim=params['clim'])
plt.plot(trk['lon'], trk['lat'], 'o-k', markersize=1, zorder=10)
plt.plot(track_nhctrk['lon'],
track_nhctrk['lat'],
'o-k',
markersize=1,
zorder=10)
plt.plot(track_higherSpeed['lon'],
track_higherSpeed['lat'],
'o-k',
markersize=1,
zorder=10)
plt.plot(track_lowerSpeed['lon'],
track_lowerSpeed['lat'],
'o-k',
markersize=1,
zorder=10)
plt.plot(track_shiftLeft['lon'],
track_shiftLeft['lat'],
'o-k',
markersize=1,
zorder=10)
plt.plot(track_shiftRight['lon'],
track_shiftRight['lat'],
'o-k',
markersize=1,
zorder=10)
title = 'HSOFS experimental ' + stormID + '.' + inputCycle + '.maxPS'
plt.text (params['lonlim'][0]+0.03, \
params['latlim'][0]+0.03, \
title )
plotter.save(
title,
outputPath + '/' + stormID + '.' + inputCycle + '.maxPS.maxele.png')
plt.close(f)
cwl = points_nhctrk # nhctrk stations
mod_dates = cwl['time']
# Plot time series
utcnow = dt.utcnow()
daterange = (utcnow - td(days=1), utcnow)
figureCounter = 0
print '[info]: Plotting time series...'
for n in range(len(cwl['lon'])):
print str(n).zfill(3)
if params['xlim'][0] <= cwl['lon'][n] and \
cwl['lon'][n] <= params['xlim'][1] and \
params['ylim'][0] <= cwl['lat'][n] and \
cwl['lat'][n] <= params['ylim'][1]:
mod_vals = np.squeeze(cwl['zeta'][:, n])
obs_vals = copy.deepcopy(mod_vals) #in case there is no obs
obs_dates = copy.deepcopy(mod_dates) # in case if there is no obs
stID = cwl['stations'][n].split()[0]
coops_id = stID #[2]
print coops_id
obs_coops = coops.getData(coops_id, daterange)
if len(obs_coops['values']) > 0:
obs_dates = obs_coops['dates']
obs_vals = obs_coops['values']
# Plot observations
f = plt.figure(figsize=(20, 4.5))
plt.plot(obs_dates, obs_vals, '.', color='g', label='OBS')
# Plot model
plt.plot(points_nhctrk['time'],
points_nhctrk['zeta'][:, n],
'.',
c='k',
label='nhctrk')
plt.plot(points_higherSpeed['time'],
points_higherSpeed['zeta'][:, n],
'.',
c='r',
label='higherSpeed',
markersize=1)
plt.plot(points_lowerSpeed['time'],
points_lowerSpeed['zeta'][:, n],
'.',
c='m',
label='lowerSpeed',
markersize=1)
plt.plot(points_shiftRight['time'],
points_shiftRight['zeta'][:, n],
'.',
c='b',
label='shiftRight',
markersize=1)
plt.plot(points_shiftLeft['time'],
points_shiftLeft['zeta'][:, n],
'.',
c='c',
label='shiftLeft',
markersize=1)
plt.ylim([-1.5, params['clim'][1]])
stationName = coops.getStationInfo(stID)
plt.title(stationName['name'] + ',' + stationName['state'])
plt.xlim([daterange[0], mod_dates[-1]])
plt.grid()
plt.xlabel('DATE UTC')
plt.ylabel('WL, meters LMSL')
plt.legend(bbox_to_anchor=(0.9, 0.35))
figFile = outputPath + '/' + stormID + '.' + inputCycle + '.ts-' + str(
figureCounter).zfill(3) + '.png'
plt.savefig(figFile)
plt.close(f)
figureCounter += 1