def main():
ds_ofs = dsofs_latest()
indexfile = f'docs/index.html'
if not os.path.exists('./docs'):
os.makedirs('./docs')
bucket = 'ioos-cloud-www'
storageService = S3Storage()
rho_vars = ['temp', "zeta", "salt"]
imagelist = []
for var in rho_vars:
imagename = plot_rho(ds_ofs, var, s3upload=True)
imagelist.append(imagename)
make_indexhtml(indexfile, imagelist)
storageService.uploadFile(indexfile,
bucket,
'index.html',
public=True,
text=True)
print('Finished ...')
def plot_rho(ds, variable, s3upload=False) -> str:
if variable == 'zeta':
da = ds[variable].isel(ocean_time=0)
cmap = cmocean.cm.phase
if variable == 'temp':
da = ds[variable].isel(ocean_time=0, s_rho=0)
cmap = cmocean.cm.thermal
if variable == 'salt':
da = ds[variable].isel(ocean_time=0, s_rho=0)
cmap = cmocean.cm.haline
if variable == 'oxygen':
da = ds[variable].isel(ocean_time=0, s_rho=0)
cmap = cmocean.cm.oxy
if variable == 'Pair':
da = ds[variable].isel(ocean_time=0)
cmap = cmocean.cm.diff
fig = plt.figure(figsize=(12,5))
ax = fig.add_axes([0,0,1,1], projection=ccrs.PlateCarree())
im = ax.contourf(da.lon_rho, da.lat_rho, da.values,
transform=ccrs.PlateCarree(),
cmap=cmap)
coast_10m = cfeature.NaturalEarthFeature(
'physical', 'land', '10m',
edgecolor='k', facecolor='0.8'
)
ax.add_feature(coast_10m);
title = ds.attrs['title']
history = ds.history
now = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
ax.set_title(f"Image generated on {now}\n\n{title}\n{history}");
cbar = fig.colorbar(im, ax=ax)
long_name = da.attrs['long_name']
if variable != 'salt':
units = da.attrs['units']
cbar.set_label(f'{long_name} ({units})')
else:
cbar.set_label(f'{long_name}')
indexfile = f'docs/index.html'
outfile = f'docs/{variable}.png'
if not os.path.exists('./docs'):
os.makedirs('./docs')
imagename = outfile.split('/')[-1]
plt.savefig(outfile, bbox_inches='tight')
if s3upload:
s3 = S3Storage()
bucket = 'ioos-cloud-www'
s3.uploadFile(outfile, bucket, f'{variable}.png', public = True)
return imagename
def main():
print('... About to load dataset ds_ofs ...')
ds_ofs = dsofs_curr_fcst()
print('... ds_ofs is loaded ...')
model = get_model_type(ds_ofs)
print(f'... model type is: {model}')
indexfile = f'docs/index.html'
if not os.path.exists('./docs'):
os.makedirs('./docs')
bucket = 'ioos-cloud-www'
bucket_folder = 'sandbot/'
storageService = S3Storage()
# TODO: vectors not working for LEOFS - hangs. NGOFS works fine.
plot_vars = ['temp', 'zeta', 'salt', 'wind', 'currents']
print(f'... Plotting these variables: {plot_vars}')
imagelist = []
for var in plot_vars:
print(f'... calling plot routine for {model}:{var} ...')
#imagename = plot_runner(ds_ofs, var, s3upload=upload)
if model == 'roms':
imagename = plot_roms(ds_ofs, var, s3upload=upload)
elif model == 'fvcom':
imagename = plot_fvcom(ds_ofs, var, s3upload=upload)
else:
print('ERROR: model not supported')
raise Exception(e)
if imagename:
imagelist.append(imagename)
ds_ofs.close()
if upload:
make_indexhtml(indexfile, imagelist)
storageService.uploadFile(indexfile,
bucket,
f'{bucket_folder}index.html',
public=True,
text=True)
print('Finished ...')
def inject(pyfile, platform: str = 'AWS'):
''' Uploads the notebook to S3 '''
#from cloudflow.services.StorageService import StorageService
from cloudflow.services.S3Storage import S3Storage
#FIX: Make this cloud agnostic, need a way to set the current platform at runtime
#FIX: Make this configurable at runtime, maybe from a config file on the machine
user = os.getenv("JUPYTERHUB_USER")
bucket = 'ioos-cloud-sandbox'
bcktfolder = f'cloudflow/inject/{user}'
filename = pyfile.split('/')[-1]
#print(f"filename: {filename}")
ss = S3Storage()
#def uploadFile(self, filename: str, bucket: str, key: str, public: bool = False):
ss.uploadFile(pyfile, bucket, f"{bcktfolder}/{filename}")
return
def main(job: Plotting):
COMDIR = job.INDIR
OFS = job.OFS
HH = job.HH
rho_vars = job.VARS
print(f'COMDIR is: {COMDIR}')
print(f'OFS is: {OFS}')
print(f'HH is: {HH}')
print('Running ...')
# could check that this is a roms model
# if ofs in utils.roms_models then do roms
# else if ofs in utils.fvcom_models then do fvcom
ds_roms = roms_nosofs(COMDIR, OFS, HH)
indexfile = f'docs/index.html'
if not os.path.exists('./docs'):
os.makedirs('./docs')
bucket = 'ioos-cloud-www'
storageService = S3Storage()
#rho_vars = ['temp',"zeta", "salt" ]
imagelist = []
for var in rho_vars:
imagename = plot_rho(ds_roms, var, s3upload=True)
imagelist.append(imagename)
make_indexhtml(indexfile, imagelist)
storageService.uploadFile(indexfile,
bucket,
'index.html',
public=True,
text=True)
print('Finished ...')
def storage_init(provider: str) -> StorageService:
"""Class factory that returns an implementation of StorageService.
StorageService is the abstract base class that provides a generic interface for
multiple cloud platforms.
Parameters
----------
provider : str
Name of an implemented provider.
Returns
-------
service : StorageService
Returns a specific implementation of the StorageService interface.
Raises
------
signals.FAIL
Triggers and exception if `provider` is not supported.
Notes
-----
The following providers are implemented:
AWS S3 - S3Storage
"""
if provider == 'AWS':
service = S3Storage()
elif provider == 'Local':
log.error('Coming soon ...')
raise signals.FAIL()
else:
log.error('Unsupported provider')
raise signals.FAIL()
return service
def plot_fvcom(ds, variable, s3upload=False) -> str:
time = 2
siglay = 0
dims = 2
clevs = 15
vector = False
vmin = None
vmax = None
print(f'.... Plotting FVCOM - {variable} ....')
if variable == 'zeta':
#zeta(time, node)
dims = 2
da = ds[variable]
#cmap = cmocean.cm.phase
#cmap=plt.get_cmap('jet')
cmap = plt.get_cmap('turbo')
#vmax=5.0
#vmin=-3.0
if variable == 'temp':
#temp(time, siglay, node)
dims = 3
#da = ds[variable][:][siglay]
da = ds.variables[variable]
cmap = cmocean.cm.thermal
if variable == 'salt':
#salinity(time, siglay, node)
dims = 3
da = ds['salinity']
cmap = cmocean.cm.haline
if variable == 'Pair':
#atmos_press(time, node)
dims = 2
da = ds['atmos_press']
cmap = cmocean.cm.diff
if variable == 'oxygen':
print('oxygen is not supported in this model')
return
if variable == 'wind':
vector = True
dims = 2
cmap = plt.get_cmap('OrRd')
#uwind_speed(time, nele)
#vwind_speed(time, nele)
data_u = ds['uwind_speed']
data_v = ds['vwind_speed']
da = data_u
long_name = 'Wind Speed'
print('.... computing wind magnitude .... ')
mag = np.sqrt(np.square(data_u[time]) + np.square(data_v[time]))
if variable == 'currents':
vector = True
dims = 3
cmap = plt.get_cmap('YlOrBr')
#u(time, siglay, nele)
#v(time, siglay, nele)
data_u = ds['u'][siglay]
data_v = ds['v'][siglay]
da = ds['u']
mag = np.sqrt(np.square(data_u[time]) + np.square(data_v[time]))
long_name = 'Water Velocity'
fig = plt.figure(figsize=(12, 5), dpi=72)
#fig = plt.figure(figsize=(12,5), facecolor='black')
ax = fig.add_axes([0, .085, 1, 1], projection=ccrs.PlateCarree())
if dims == 2:
plot_da = da[time]
elif dims == 3:
plot_da = da[time][siglay]
else:
plot_da = None
if debug: print(f"{variable} dims: {dims}")
if vector == False:
lon = ds['lon'][:]
lon = np.where(lon > 180., lon - 360., lon)
lat = ds['lat'][:]
nv = ds.variables['nv'][:].T
nv = nv - 1
im = ax.tricontourf(lon,
lat,
nv,
plot_da,
transform=ccrs.PlateCarree(),
levels=clevs,
vmin=vmin,
vmax=vmax,
cmap=cmap)
long_name = da.long_name
else:
# Plot the magnitude
lon = ds['lonc'][:]
lon = np.where(lon > 180., lon - 360., lon)
lat = ds['latc'][:]
#nbe(three, nele) ;
#nbe:long_name = "elements surrounding each element" ;
nbe = ds.variables['nbe'][:].T
nbe = nbe - 1
# Think of it as a circular array, need to keep vertices together
np.place(nbe, nbe < 0, len(nbe) - 1)
im = ax.tricontourf(lon,
lat,
nbe,
mag,
transform=ccrs.PlateCarree(),
levels=clevs,
vmin=vmin,
vmax=vmax,
cmap=cmap)
# Plot the vectors
im = ax.quiver(lon,
lat,
data_u[time],
data_v[time],
cmap=cmap,
regrid_shape=18)
# end if vector
print('.... adding map features and labels ....')
# Don't add the land mask if it is a Lake forecast
if not (re.search("LEOFS", ds.title) or re.search("LMHOFS", ds.title)):
# if not re.search("Lake", ds.title):
# Obscures data
#if variable == 'temp':
land = cfeature.NaturalEarthFeature('physical',
'land',
'10m',
edgecolor='black',
facecolor='0.75')
ax.add_feature(land)
else:
# We currently aren't running any saltwater lakes
if variable == "salt":
print('INFO: Not plotting salinity for freshwater lakes ....')
plt.close(fig)
return ""
lakes = cfeature.NaturalEarthFeature('physical',
'lakes',
'10m',
edgecolor='black',
facecolor='none')
ax.add_feature(lakes)
# Does not have lake contour
# ax.coastlines(resolution='10m')
# OSM data example
#shp = shapereader.Reader('./data/OSM_land/BTS')
#for record, geometry in zip(shp.records(), shp.geometries()):
# ax.add_geometries([geometry], ccrs.PlateCarree(), facecolor='lightgray',
# edgecolor='black')
title = ds.title
time_array = ds.variables['time']
init = time_array[0]
#init=ds.dstart.isel(ocean_time=otime) # init is: <xarray.DataArray 'dstart' (ocean_time: 2)>
#init_datetime = num2date(init, init.units) + datetime.timedelta(hours=6)
#init_str = f"INIT: {init_datetime} UTC"
init_str = f"INIT: {num2date(init, time_array.units)} UTC"
valid = time_array[time]
valid_str = f"VALID: {num2date(valid, time_array.units)} UTC"
ax.set_title(f'{title}\n{init_str} {valid_str}')
# COLOR BAR
if vmin:
cm = plt.cm.ScalarMappable(cmap=cmap)
cm.set_clim(vmin, vmax)
cbar = fig.colorbar(cm, ax=ax, pad=0.02)
else:
cbar = fig.colorbar(im, ax=ax, pad=0.02)
if variable != 'salt':
units = da.units
cbar.set_label(f'{long_name} ({units})')
else:
cbar.set_label(f'{long_name}')
# Not easy to get the local system timezone info using python
# https://stackoverflow.com/questions/35057968/get-system-local-timezone-in-python/35058346
dst = timelib.localtime().tm_isdst > 0
if dst: tz = 'EDT'
else: tz = 'EST'
datestrfmt = '%b %d, %Y %H:%M %Z' #'%Y-%m-%d %H:%M:%S'
now_str = f"FVCOM plotted at {datetime.datetime.now().strftime(datestrfmt)} {tz}"
fig.text(0.65, 0.02, f'{now_str}')
print('.... adding logos ....')
bbox = ax.get_position().bounds
#img = image.imread('rps_small.png')
logo1 = io.BytesIO(base64.b64decode(rps_logo))
logo1 = image.imread(logo1, format='PNG')
logo1_axis = fig.add_axes([bbox[0], 0.0, 0.1, 0.08])
logo1_axis.imshow(logo1, interpolation='hanning')
logo1_axis.axis('off')
logo2 = io.BytesIO(base64.b64decode(ioos_logo))
logo2 = image.imread(logo2, format='PNG')
logo2_axis = fig.add_axes([bbox[0], 0.0, 0.325, 0.08])
logo2_axis.imshow(logo2, interpolation='hanning')
logo2_axis.axis('off')
# Add the forecast date and time to the filename
init_datetime = num2date(init, time_array.units)
fmt = '%Y%m%d_%HZ' #'%Y-%m-%d %H:%M:%S'
ini_str = f"{init_datetime.strftime(fmt)}"
indexfile = f'docs/index.html'
outfile = f'docs/{ini_str}_{variable}.png'
if not os.path.exists('./docs'):
os.makedirs('./docs')
imagename = outfile.split('/')[-1]
print(f'.... saving image - {outfile} ....')
plt.savefig(outfile, bbox_inches='tight')
if s3upload:
print(f'.... uploading image to S3 - {variable} ....')
s3 = S3Storage()
bucket = 'ioos-cloud-www'
bucket_folder = 'sandbot/'
key = f'{bucket_folder}{imagename}'
s3.uploadFile(outfile, bucket, key, public=True)
return imagename
def plot_roms(ds, variable, s3upload=False) -> str:
otime = 1
srho = len(ds.s_rho) - 1
# print(f"DEBUG: srho is {srho}")
vmin = None
vmax = None
if variable == 'zeta':
da = ds[variable].isel(ocean_time=otime)
#cmap = cmocean.cm.balance
cmap = plt.get_cmap('jet')
vector = False
vmin = -20.0
vmax = 20.0
if variable == 'temp':
da = ds[variable].isel(ocean_time=otime,
s_rho=srho) #s_rho=0 = surface
cmap = cmocean.cm.thermal
vector = False
if variable == 'salt':
#salt(ocean_time, s_rho, eta_rho, xi_rho)
da = ds[variable].isel(ocean_time=otime, s_rho=srho)
cmap = cmocean.cm.haline
vector = False
if variable == 'oxygen':
da = ds[variable].isel(ocean_time=otime, s_rho=srho)
cmap = cmocean.cm.oxy
vector = False
if variable == 'Pair':
# Pair(ocean_time, eta_rho, xi_rho)
da = ds[variable].isel(ocean_time=otime)
cmap = cmocean.cm.diff
vector = False
if variable == 'wind':
cmap = plt.get_cmap('OrRd')
# Uwind(ocean_time, eta_rho, xi_rho) ;
# Vwind(ocean_time, eta_rho, xi_rho)
data_u = ds['Uwind'].isel(ocean_time=otime)
data_v = ds['Vwind'].isel(ocean_time=otime)
lons = ds['lon_rho'].values
lats = ds['lat_rho'].values
da = data_u
u = ma.masked_invalid(data_u.values)
v = ma.masked_invalid(data_v.values)
mag = np.sqrt(np.square(u) + np.square(v))
vector = True
if variable == 'currents':
cmap = plt.get_cmap('YlOrBr')
# Select only the ocean time needed.
# Select u/v data at layer srho (last index = water surface)
u = ds.u.values[otime, srho, :, :]
v = ds.v.values[otime, srho, :, :]
#print(f'u before (ds.u.shape): {u.shape}')
#print(f'v before (ds.v.shape): {v.shape}')
# Interpolate u/v to rho coordinates
ds['u_rho'] = (('eta_rho', 'xi_rho'),
np.ma.masked_values(
np.zeros_like(ds.zeta.isel(ocean_time=otime)), 0.0))
ds['v_rho'] = (('eta_rho', 'xi_rho'),
np.ma.masked_values(
np.zeros_like(ds.zeta.isel(ocean_time=otime)), 0.0))
ds.u_rho[1:, 1:] = (u[0:-1, :] + u[1:, :]) * 0.5
ds.v_rho[1:, 1:] = (v[:, 0:-1] + v[:, 1:]) * 0.5
#print(f'u after (ds.u_rho.shape): {ds.u_rho.shape}')
#print(f'v after (ds.v_rho.shape): {ds.v_rho.shape}')
data_u = ds['u_rho']
data_v = ds['v_rho']
da = ds['u'].isel(ocean_time=otime)[srho, :, :]
u = data_u.values
v = data_v.values
lons = ds['lon_rho'].values
lats = ds['lat_rho'].values
mag = np.sqrt(np.square(u) + np.square(v))
vector = True
# end if currents
fig = plt.figure(figsize=(12, 5), dpi=72)
ax = fig.add_axes([0, 0.1, 1, 1], projection=ccrs.PlateCarree())
maskrho = ds['mask_rho'].isel(ocean_time=otime)
if not vector:
# Normal plot
vari = da.values
vari[np.where(maskrho == 0)] = np.nan
im = ax.contourf(da.lon_rho,
da.lat_rho,
vari,
transform=ccrs.PlateCarree(),
levels=12,
vmin=vmin,
vmax=vmax,
cmap=cmap)
## VECTORS!
else:
# Plot the magnitude
vari = mag
vari[np.where(maskrho == 0)] = np.nan
u[np.where(maskrho == 0)] = np.nan
v[np.where(maskrho == 0)] = np.nan
im = ax.contourf(lons,
lats,
vari,
transform=ccrs.PlateCarree(),
levels=12,
cmap=cmap)
# Plot the quivers
im = ax.quiver(lons,
lats,
u,
v,
cmap=cmap,
transform=ccrs.PlateCarree(),
regrid_shape=32)
coast_10m = cfeature.NaturalEarthFeature('physical',
'land',
'10m',
edgecolor='k',
facecolor='0.8')
ax.add_feature(coast_10m)
#init = ds.ocean_time.isel(ocean_time=0) - Not always right
init = ds.dstart.isel(
ocean_time=otime
) # init is: <xarray.DataArray 'dstart' (ocean_time: 2)>
init_datetime = num2date(init, init.units) + datetime.timedelta(hours=6)
init_str = f"INIT: {init_datetime} UTC"
valid = da.ocean_time
valid_str = f"VALID: {num2date(valid, valid.units)} UTC"
title = ds.title
ax.set_title(f'{title}\n{init_str} {valid_str}')
if vmin:
cm = plt.cm.ScalarMappable(cmap=cmap)
cm.set_clim(vmin, vmax)
cbar = fig.colorbar(cm, ax=ax, pad=0.02)
else:
cbar = fig.colorbar(im, ax=ax, pad=0.02)
if vector == False:
long_name = da.attrs['long_name']
else:
long_name = variable
if variable != 'salt':
units = da.attrs['units']
cbar.set_label(f'{long_name} ({units})')
else:
cbar.set_label(f'{long_name}')
# Add the forecast date and time to the filename
fmt = '%Y%m%d_%HZ' #'%Y-%m-%d %H:%M:%S'
ini_str = f"{init_datetime.strftime(fmt)}"
indexfile = f'docs/index.html'
outfile = f'docs/{ini_str}_{variable}.png'
bbox = ax.get_position().bounds
#img = image.imread('rps_small.png')
logo1 = io.BytesIO(base64.b64decode(rps_logo))
logo1 = image.imread(logo1, format='PNG')
logo1_axis = fig.add_axes([bbox[0], 0.0, 0.1, 0.08])
logo1_axis.imshow(logo1, interpolation='hanning')
logo1_axis.axis('off')
logo2 = io.BytesIO(base64.b64decode(ioos_logo))
logo2 = image.imread(logo2, format='PNG')
logo2_axis = fig.add_axes([bbox[0], 0.0, 0.325, 0.08])
logo2_axis.imshow(logo2, interpolation='hanning')
logo2_axis.axis('off')
dst = timelib.localtime().tm_isdst > 0
if dst: tz = 'EDT'
else: tz = 'EST'
datestrfmt = '%b %d, %Y %H:%M %Z' #'%Y-%m-%d %H:%M:%S'
#https://docs.python.org/3/library/datetime.html#aware-and-naive-objects
now_str = f"ROMS plotted at {datetime.datetime.now().strftime(datestrfmt)} {tz}"
fig.text(0.65, 0.02, f'{now_str}')
if not os.path.exists('./docs'):
os.makedirs('./docs')
imagename = outfile.split('/')[-1]
plt.savefig(outfile, bbox_inches='tight')
#plt.close()
if s3upload:
s3 = S3Storage()
bucket = 'ioos-cloud-www'
bucket_folder = 'sandbot/'
key = f'{bucket_folder}{imagename}'
s3.uploadFile(outfile, bucket, key, public=True)
return imagename