def convert(ipath, opath, variable, ncore, meshpath, skip):
mesh = pf.load_mesh(meshpath, usepickle=False, usejoblib=True)
ind_depth_all = []
ind_noempty_all = []
ind_empty_all = []
for i in range(len(mesh.zlevs)):
ind_depth, ind_noempty, ind_empty = pf.ind_for_depth(
mesh.zlevs[i], mesh)
ind_depth_all.append(ind_depth)
ind_noempty_all.append(ind_noempty)
ind_empty_all.append(ind_empty)
Parallel(n_jobs=ncore, verbose=50)(
delayed(f2l)(l, opath, variable, mesh, skip, ind_noempty_all,
ind_empty_all, ind_depth_all) for l in ipath)
def load_mesh(config):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(config["host"], username=config["user"])
with ssh.open_sftp() as sftp:
scriptdir_contents = sftp.listdir(config["basedir"] + "/scripts")
runscript = (config["basedir"] + "/scripts/" +
[f for f in scriptdir_contents if f.endswith(".run")][0])
with sftp.file(runscript) as runscript_file:
mesh_dir = [
l.strip() for l in runscript_file.readlines()
if "MESH_DIR" in l
][0].split("=")[-1]
os.system("rsync -azv " + config["user"] + "@" + config["host"] + ":" +
mesh_dir + " " + config["storagedir"] + "/" +
config["model"] + "/MESHES/")
return pf.load_mesh(config["storagedir"] + "/" + config["model"] +
"/MESHES/" + mesh_dir.split("/")[-1])
import sys
sys.path.append("/home/h/hbkqtang/pyfesom.git/trunk/")
import pyfesom as pf
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from matplotlib.colors import LinearSegmentedColormap
import numpy as np
from matplotlib import cm
from netCDF4 import Dataset
meshpath='/gfs1/work/hbkqtang/input/CORE2_final/'
mesh=pf.load_mesh(meshpath,usepickle=False)
m = Basemap(projection='robin',lon_0=0, resolution='c')
x, y = m(mesh.x2, mesh.y2)
# Read the observation data
fl_obs=Dataset('/gfs1/work/hbkqtang/temporary/obs_SST.nc')
# Read the model forecast from the free run
fl_f=Dataset('/gfs1/work/hbkqtang/output/cpl_work_2014_3/thetao_fesom_20140101.nc')
for i in range(730,1096):
level_data_obs, elem_no_nan = pf.get_data(fl_obs.variables['sst'][i-730,:],mesh,000)
level_data_f, elem_no_nan_dummy = pf.get_data(fl_f.variables['thetao'][i,:],mesh,000)
# Calculate the differnece
level_data=level_data_obs-level_data_f
# Plot the figure
fig=plt.figure(figsize=(10,7))
m.drawmapboundary(fill_color='0.9')
m.drawcoastlines()
def showfile(ifile, variable, depth, meshpath, box, res, influence, timestep,
levels, quiet, ofile, mapproj, abg, clim, cmap, interp, ptype, k):
'''
meshpath - Path to the folder with FESOM1.4 mesh files.
ifile - Path to FESOM1.4 netCDF file.
variable - The netCDF variable to be plotted.
'''
if not quiet:
click.secho('Mesh: {}'.format(meshpath))
click.secho('File: {}'.format(ifile))
click.secho('Variable: {}'.format(variable), fg='red')
click.secho('Depth: {}'.format(depth), fg='red')
click.secho('BOX: {}'.format(box))
click.secho('Resolution: {}'.format(res))
click.secho('Influence raduis: {} meters'.format(influence), fg='red')
click.secho('Timestep: {}'.format(timestep))
if levels:
click.secho('Levels: {}'.format(levels), fg='red')
else:
click.secho('Levels: auto', fg='red')
if cmap:
if cmap in cmo.cmapnames:
colormap = cmo.cmap_d[cmap]
elif cmap in plt.cm.datad:
colormap = plt.get_cmap(cmap)
else:
raise ValueError(
'Get unrecognised name for the colormap `{}`. Colormaps should be from standard matplotlib set of from cmocean package.'
.format(cmap))
else:
if clim:
colormap = cmo.cmap_d['balance']
else:
colormap = plt.get_cmap('Spectral_r')
sstep = timestep
radius_of_influence = influence
left, right, down, up = box
lonNumber, latNumber = res
mesh = pf.load_mesh(meshpath, abg=abg, usepickle=False, usejoblib=True)
flf = Dataset(ifile)
lonreg = np.linspace(left, right, lonNumber)
latreg = np.linspace(down, up, latNumber)
lonreg2, latreg2 = np.meshgrid(lonreg, latreg)
dind = (abs(mesh.zlevs - depth)).argmin()
realdepth = mesh.zlevs[dind]
level_data, nnn = pf.get_data(flf.variables[variable][sstep], mesh,
realdepth)
if interp == 'nn':
ofesom = pf.fesom2regular(level_data,
mesh,
lonreg2,
latreg2,
radius_of_influence=radius_of_influence)
elif interp == 'idist':
ofesom = pf.fesom2regular(level_data,
mesh,
lonreg2,
latreg2,
radius_of_influence=radius_of_influence,
how='idist',
k=k)
elif interp == 'linear':
points = np.vstack((mesh.x2, mesh.y2)).T
qh = qhull.Delaunay(points)
ofesom = LinearNDInterpolator(qh, level_data)((lonreg2, latreg2))
elif interp == 'cubic':
points = np.vstack((mesh.x2, mesh.y2)).T
qh = qhull.Delaunay(points)
ofesom = CloughTocher2DInterpolator(qh, level_data)((lonreg2, latreg2))
if clim:
if variable == 'temp':
climvar = 'T'
elif variable == 'salt':
climvar = 'S'
else:
raise ValueError(
'You have selected --clim/-c option, but variable `{}` is not in climatology. Acceptable values are `temp` and `salt` only.'
.format(variable))
#os.path.join(os.path.dirname(__file__), "../")
pathToClim = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../data/")
print(pathToClim)
w = pf.climatology(pathToClim, clim)
xx, yy, oclim = pf.clim2regular(
w,
climvar,
lonreg2,
latreg2,
levels=[realdepth],
radius_of_influence=radius_of_influence)
oclim = oclim[0, :, :]
data = ofesom - oclim
else:
data = ofesom
if mapproj == 'merc':
ax = plt.subplot(111, projection=ccrs.Mercator())
elif mapproj == 'pc':
ax = plt.subplot(111, projection=ccrs.PlateCarree())
elif mapproj == 'np':
ax = plt.subplot(111, projection=ccrs.NorthPolarStereo())
elif mapproj == 'sp':
ax = plt.subplot(111, projection=ccrs.SouthPolarStereo())
elif mapproj == 'rob':
ax = plt.subplot(111, projection=ccrs.Robinson())
ax.set_extent([left, right, down, up], crs=ccrs.PlateCarree())
if levels:
mmin, mmax, nnum = levels
nnum = int(nnum)
else:
mmin = np.nanmin(data)
mmax = np.nanmax(data)
nnum = 40
data_levels = np.linspace(mmin, mmax, nnum)
if ptype == 'cf':
mm = ax.contourf(lonreg,\
latreg,\
data,
levels = data_levels,
transform=ccrs.PlateCarree(),
cmap=colormap,
extend='both')
elif ptype == 'pcm':
data_cyc, lon_cyc = add_cyclic_point(data, coord=lonreg)
mm = ax.pcolormesh(lon_cyc,\
latreg,\
data_cyc,
vmin = mmin,
vmax = mmax,
transform=ccrs.PlateCarree(),
cmap=colormap,
)
else:
raise ValueError('Inknown plot type {}'.format(ptype))
ax.coastlines(resolution='50m', lw=0.5)
ax.add_feature(
cfeature.GSHHSFeature(levels=[1], scale='low', facecolor='lightgray'))
cb = plt.colorbar(mm, orientation='horizontal', pad=0.03)
cb.set_label(flf.variables[variable].units)
plt.title('{} at {}m.'.format(variable, realdepth))
plt.tight_layout()
if ofile:
plt.savefig(ofile, dpi=100)
else:
plt.show()
def convert(meshpath, ipath, opath, variable, depths, box,
res, influence, timestep, abg, interp, ncore, k):
'''
meshpath - Path to the folder with FESOM1.4 mesh files.
ipath - Path to FESOM1.4 netCDF file or files (with wildcard).
opath - path where the output will be stored.
variable - The netCDF variable to be converted.
'''
print(ipath)
mesh = pf.load_mesh(meshpath, abg=abg, usepickle=False, usejoblib=True)
sstep = timestep
radius_of_influence = influence
left, right, down, up = box
lonNumber, latNumber = res
lonreg = np.linspace(left, right, lonNumber)
latreg = np.linspace(down, up, latNumber)
lonreg2, latreg2 = np.meshgrid(lonreg, latreg)
localdir = os.path.dirname(os.path.abspath(__file__))
# print(os.path.abspath(__file__))
print('localdir='+localdir)
with open(localdir+'/CMIP6_Omon.json') as data_file:
cmore_table = json.load(data_file, object_pairs_hook=OrderedDict)
with open(localdir+'/CMIP6_SIday.json') as data_file:
cmore_table_ice = json.load(data_file, object_pairs_hook=OrderedDict)
depths = np.array(depths.split(' '),dtype='float32')
if depths[0] == -1:
dind = range(mesh.zlevs.shape[0])
realdepth = mesh.zlevs
else:
dind = []
realdepth = []
for depth in depths:
ddepth = abs(mesh.zlevs-depth).argmin()
dind.append(ddepth)
realdepth.append(mesh.zlevs[ddepth])
print(dind)
print(realdepth)
#realdepth = mesh.zlevs[dind]
distances, inds = pf.create_indexes_and_distances(mesh, lonreg2, latreg2,\
k=k, n_jobs=4)
ind_depth_all = []
ind_noempty_all = []
ind_empty_all = []
for i in range(len(mesh.zlevs)):
ind_depth, ind_noempty, ind_empty = pf.ind_for_depth(mesh.zlevs[i], mesh)
ind_depth_all.append(ind_depth)
ind_noempty_all.append(ind_noempty)
ind_empty_all.append(ind_empty)
if interp == 'nn':
topo_interp = pf.fesom2regular(mesh.topo, mesh, lonreg2, latreg2, distances=distances,
inds=inds, radius_of_influence=radius_of_influence, n_jobs=1)
k = 1
distances, inds = pf.create_indexes_and_distances(mesh, lonreg2, latreg2,\
k=k, n_jobs=4)
points, qh = None, None
elif interp == 'idist':
topo_interp = pf.fesom2regular(mesh.topo, mesh, lonreg2, latreg2, distances=distances,
inds=inds, radius_of_influence=radius_of_influence, n_jobs=1, how='idist')
k = k
distances, inds = pf.create_indexes_and_distances(mesh, lonreg2, latreg2,\
k=k, n_jobs=4)
points, qh = None, None
elif interp == 'linear':
points = np.vstack((mesh.x2, mesh.y2)).T
qh = qhull.Delaunay(points)
topo_interp = LinearNDInterpolator(qh, mesh.topo)((lonreg2, latreg2))
distances, inds = None, None
elif interp == 'cubic':
points = np.vstack((mesh.x2, mesh.y2)).T
qh = qhull.Delaunay(points)
topo_interp = CloughTocher2DInterpolator(qh, mesh.topo)((lonreg2, latreg2))
distances, inds = None, None
mdata = maskoceans(lonreg2, latreg2, topo_interp, resolution = 'h', inlands=False)
topo = np.ma.masked_where(~mdata.mask, topo_interp)
# Backend is switched to threading for linear and cubic interpolations
# due to problems with memory mapping.
# One have to test threading vs multiprocessing.
if (interp == 'linear') or (interp=='cubic'):
backend = 'threading'
else:
backend = 'multiprocessing'
Parallel(n_jobs=ncore, backend=backend, verbose=50)(delayed(scalar2geo)(ifile, opath, variable,
mesh, ind_noempty_all,
ind_empty_all,ind_depth_all, cmore_table, lonreg2, latreg2,
distances, inds, radius_of_influence, topo, points, interp, qh, timestep, dind, realdepth) for ifile in ipath)
def showo(meshpath, ifile, variable, depth, box, timestep, minmax, mapproj,
quiet, ofile):
'''
meshpath - Path to the folder with FESOM1.4 mesh files.
ifile - Path to FESOM1.4 netCDF file.
variable - The netCDF variable to be plotted.
'''
if not quiet:
click.secho('Mesh: {}'.format(meshpath))
click.secho('File: {}'.format(ifile))
click.secho('Variable: {}'.format(variable), fg='red')
click.secho('Depth: {}'.format(depth), fg='red')
click.secho('BOX: {}'.format(box))
click.secho('Timestep: {}'.format(timestep))
if minmax:
click.secho('Min/max: {}'.format(minmax), fg='red')
else:
click.secho('Min/max: auto', fg='red')
mesh = pf.load_mesh(meshpath)
flf = Dataset(ifile)
left, right, down, up = box
elem_nonan, no_nan_tri = pf.cut_region(mesh, [left, right, down, up], 0)
if variable == 'topo':
level_data = mesh.topo
else:
level_data, nnn = pf.get_data(flf.variables[variable][timestep, :],
mesh, depth)
if minmax:
mmin, mmax = minmax
else:
mmin = level_data[elem_nonan].min()
mmax = level_data[elem_nonan].max()
plt.figure(figsize=(10, 10))
if mapproj == 'merc':
ax = plt.subplot(111, projection=ccrs.Mercator())
elif mapproj == 'pc':
ax = plt.subplot(111, projection=ccrs.PlateCarree())
elif mapproj == 'np':
ax = plt.subplot(111, projection=ccrs.NorthPolarStereo())
elif mapproj == 'sp':
ax = plt.subplot(111, projection=ccrs.SouthPolarStereo())
elif mapproj == 'rob':
ax = plt.subplot(111, projection=ccrs.Robinson())
ax.set_extent(box, crs=ccrs.PlateCarree())
mm = ax.tripcolor(mesh.x2,
mesh.y2,
elem_nonan,
level_data,
transform=ccrs.PlateCarree(),
edgecolors='k',
cmap=cm.Spectral_r,
vmin=mmin,
vmax=mmax)
ax.coastlines(lw=0.5, resolution='10m')
plt.colorbar(
mm,
orientation='horizontal',
pad=0.03,
)
plt.title('{} at {}'.format(variable, depth), size=20)
if ofile:
plt.savefig(ofile, dpi=200)
else:
plt.show()
sys.argv[3]), int(sys.argv[4])
#################################################################################################
# mesh-specific configs
if meshname4plots == 'LR':
meshpath = '/work/bm0944/input/CORE2_final/' # COREII at DKRZ
savepath = '/pf/a/a270046/hierarchy/new_figures/paper-animation/meridional_ATL/LR/'
#filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output/fesom.{}.oce.mean.nc' # original COREII data at DKRZ
filedia = '/work/ab0995/a270046/hierarchy-grids/cpl_output/fesom.{}.oce.diag.nc' # COREII data at DKRZ
filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output/cdo-postprocessing/fesom.{}.oce.annualmean.nc' # annual COREII
# load mesh
mesh = pf.load_mesh(meshpath, get3d=True, usepickle=False, usejoblib=True)
elif meshname4plots == 'REF':
meshpath = '/work/bm0944/input/mesh_ref87k/' # REF at DKRZ
savepath = '/pf/a/a270046/hierarchy/new_figures/paper-animation/meridional_ATL/REF/'
#filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output_ref87k/fesom.{}.oce.mean.nc' # original REF data at DKRZ
filedia = '/work/ab0995/a270046/hierarchy-grids/cpl_output_ref87k/fesom.{}.oce.diag.nc' # REF data at DKRZ
filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output_ref87k/cdo-postprocessing/fesom.{}.oce.annualmean.nc' # annual REF
mesh = pf.load_mesh(meshpath, get3d=True, usepickle=False, usejoblib=True)
elif meshname4plots == 'MR0':
meshpath = '/work/bm0944/input/aguv/' # AGUV at DKRZ
end_year = config.getint('main', 'end_year')
ifile_template = config.get('main', 'ifile_template')
ifile_template_ice = config.get('main', 'ifile_template_ice')
ofile_template = config.get('main', 'ofile_template')
distribute_timesteps = config.getboolean('main', 'distribute_timesteps')
# Generate regular grid
lon = np.linspace(left_lon, right_lon, number_of_lons)
lat = np.linspace(lower_lat, upper_lat, number_of_lats)
lons, lats = np.meshgrid(lon, lat)
# read the FESOM mesh
print('=' * 50)
mesh = pf.load_mesh(meshpath, abg=angles_for_mesh, get3d=True, usepickle=True)
# Open CMOR variable descriptions
with open('CMIP6_Omon.json') as data_file:
cmore_table = json.load(data_file, object_pairs_hook=OrderedDict)
with open('CMIP6_SIday.json') as data_file:
cmore_table_ice = json.load(data_file, object_pairs_hook=OrderedDict)
# Add some variables that are missing in CMOR tables
cmore_table['variable_entry']['wo'] = OrderedDict([
(u'modeling_realm', u'ocean'), (u'standard_name', u'sea_water_z_velocity'),
(u'units', u'm s-1'), (u'cell_methods', u'time: mean'),
(u'cell_measures', u'--OPT'), (u'long_name', u'Sea Water Z Velocity'),
(u'comment', u'Not standard CMORE variable'),
(u'dimensions', u'longitude latitude olevel time'), (u'out_name', u'wo'),
def load_mesh(mesh_path):
"""Wrapper function to load the mesh with timing"""
return pf.load_mesh(mesh_path, usepickle=False)
def loadmeshdata(meshpath, abg):
mesh = pf.load_mesh(meshpath, abg=abg, usepickle=False, usejoblib=True)
return mesh
