Exemplo n.º 1
0
def convert(ipath, opath, variable, ncore, meshpath, skip):

    mesh = pf.load_mesh(meshpath, usepickle=False, usejoblib=True)

    ind_depth_all = []
    ind_noempty_all = []
    ind_empty_all = []

    for i in range(len(mesh.zlevs)):
        ind_depth, ind_noempty, ind_empty = pf.ind_for_depth(
            mesh.zlevs[i], mesh)
        ind_depth_all.append(ind_depth)
        ind_noempty_all.append(ind_noempty)
        ind_empty_all.append(ind_empty)

    Parallel(n_jobs=ncore, verbose=50)(
        delayed(f2l)(l, opath, variable, mesh, skip, ind_noempty_all,
                     ind_empty_all, ind_depth_all) for l in ipath)
Exemplo n.º 2
0
def load_mesh(config):
    ssh = paramiko.SSHClient()
    ssh.load_system_host_keys()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(config["host"], username=config["user"])
    with ssh.open_sftp() as sftp:
        scriptdir_contents = sftp.listdir(config["basedir"] + "/scripts")
        runscript = (config["basedir"] + "/scripts/" +
                     [f for f in scriptdir_contents if f.endswith(".run")][0])
        with sftp.file(runscript) as runscript_file:
            mesh_dir = [
                l.strip() for l in runscript_file.readlines()
                if "MESH_DIR" in l
            ][0].split("=")[-1]
        os.system("rsync -azv " + config["user"] + "@" + config["host"] + ":" +
                  mesh_dir + " " + config["storagedir"] + "/" +
                  config["model"] + "/MESHES/")
    return pf.load_mesh(config["storagedir"] + "/" + config["model"] +
                        "/MESHES/" + mesh_dir.split("/")[-1])
Exemplo n.º 3
0
import sys
sys.path.append("/home/h/hbkqtang/pyfesom.git/trunk/")
import pyfesom as pf
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from matplotlib.colors import LinearSegmentedColormap
import numpy as np
from matplotlib import cm
from netCDF4 import Dataset
meshpath='/gfs1/work/hbkqtang/input/CORE2_final/'
mesh=pf.load_mesh(meshpath,usepickle=False)
m = Basemap(projection='robin',lon_0=0, resolution='c')
x, y = m(mesh.x2, mesh.y2)

# Read the observation data
fl_obs=Dataset('/gfs1/work/hbkqtang/temporary/obs_SST.nc')

# Read the model forecast from the free run
fl_f=Dataset('/gfs1/work/hbkqtang/output/cpl_work_2014_3/thetao_fesom_20140101.nc')

for i in range(730,1096):
  level_data_obs, elem_no_nan = pf.get_data(fl_obs.variables['sst'][i-730,:],mesh,000)
  level_data_f, elem_no_nan_dummy = pf.get_data(fl_f.variables['thetao'][i,:],mesh,000)

  # Calculate the differnece 
  level_data=level_data_obs-level_data_f

  # Plot the figure
  fig=plt.figure(figsize=(10,7))
  m.drawmapboundary(fill_color='0.9')
  m.drawcoastlines()
Exemplo n.º 4
0
def showfile(ifile, variable, depth, meshpath, box, res, influence, timestep,
             levels, quiet, ofile, mapproj, abg, clim, cmap, interp, ptype, k):
    '''
    meshpath - Path to the folder with FESOM1.4 mesh files.

    ifile    - Path to FESOM1.4 netCDF file.

    variable - The netCDF variable to be plotted.
    '''
    if not quiet:
        click.secho('Mesh: {}'.format(meshpath))
        click.secho('File: {}'.format(ifile))
        click.secho('Variable: {}'.format(variable), fg='red')
        click.secho('Depth: {}'.format(depth), fg='red')
        click.secho('BOX: {}'.format(box))
        click.secho('Resolution: {}'.format(res))
        click.secho('Influence raduis: {} meters'.format(influence), fg='red')
        click.secho('Timestep: {}'.format(timestep))
        if levels:
            click.secho('Levels: {}'.format(levels), fg='red')
        else:
            click.secho('Levels: auto', fg='red')

    if cmap:
        if cmap in cmo.cmapnames:
            colormap = cmo.cmap_d[cmap]
        elif cmap in plt.cm.datad:
            colormap = plt.get_cmap(cmap)
        else:
            raise ValueError(
                'Get unrecognised name for the colormap `{}`. Colormaps should be from standard matplotlib set of from cmocean package.'
                .format(cmap))
    else:
        if clim:
            colormap = cmo.cmap_d['balance']
        else:
            colormap = plt.get_cmap('Spectral_r')

    sstep = timestep
    radius_of_influence = influence

    left, right, down, up = box
    lonNumber, latNumber = res

    mesh = pf.load_mesh(meshpath, abg=abg, usepickle=False, usejoblib=True)
    flf = Dataset(ifile)
    lonreg = np.linspace(left, right, lonNumber)
    latreg = np.linspace(down, up, latNumber)
    lonreg2, latreg2 = np.meshgrid(lonreg, latreg)

    dind = (abs(mesh.zlevs - depth)).argmin()
    realdepth = mesh.zlevs[dind]

    level_data, nnn = pf.get_data(flf.variables[variable][sstep], mesh,
                                  realdepth)
    if interp == 'nn':
        ofesom = pf.fesom2regular(level_data,
                                  mesh,
                                  lonreg2,
                                  latreg2,
                                  radius_of_influence=radius_of_influence)
    elif interp == 'idist':
        ofesom = pf.fesom2regular(level_data,
                                  mesh,
                                  lonreg2,
                                  latreg2,
                                  radius_of_influence=radius_of_influence,
                                  how='idist',
                                  k=k)
    elif interp == 'linear':
        points = np.vstack((mesh.x2, mesh.y2)).T
        qh = qhull.Delaunay(points)
        ofesom = LinearNDInterpolator(qh, level_data)((lonreg2, latreg2))

    elif interp == 'cubic':
        points = np.vstack((mesh.x2, mesh.y2)).T
        qh = qhull.Delaunay(points)
        ofesom = CloughTocher2DInterpolator(qh, level_data)((lonreg2, latreg2))

    if clim:
        if variable == 'temp':
            climvar = 'T'
        elif variable == 'salt':
            climvar = 'S'
        else:
            raise ValueError(
                'You have selected --clim/-c option, but variable `{}` is not in climatology. Acceptable values are `temp` and `salt` only.'
                .format(variable))
        #os.path.join(os.path.dirname(__file__), "../")
        pathToClim = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                  "../data/")
        print(pathToClim)
        w = pf.climatology(pathToClim, clim)
        xx, yy, oclim = pf.clim2regular(
            w,
            climvar,
            lonreg2,
            latreg2,
            levels=[realdepth],
            radius_of_influence=radius_of_influence)
        oclim = oclim[0, :, :]
        data = ofesom - oclim
    else:
        data = ofesom

    if mapproj == 'merc':
        ax = plt.subplot(111, projection=ccrs.Mercator())
    elif mapproj == 'pc':
        ax = plt.subplot(111, projection=ccrs.PlateCarree())
    elif mapproj == 'np':
        ax = plt.subplot(111, projection=ccrs.NorthPolarStereo())
    elif mapproj == 'sp':
        ax = plt.subplot(111, projection=ccrs.SouthPolarStereo())
    elif mapproj == 'rob':
        ax = plt.subplot(111, projection=ccrs.Robinson())

    ax.set_extent([left, right, down, up], crs=ccrs.PlateCarree())

    if levels:
        mmin, mmax, nnum = levels
        nnum = int(nnum)
    else:
        mmin = np.nanmin(data)
        mmax = np.nanmax(data)
        nnum = 40

    data_levels = np.linspace(mmin, mmax, nnum)
    if ptype == 'cf':
        mm = ax.contourf(lonreg,\
                     latreg,\
                     data,
                     levels = data_levels,
                     transform=ccrs.PlateCarree(),
                     cmap=colormap,
                    extend='both')
    elif ptype == 'pcm':
        data_cyc, lon_cyc = add_cyclic_point(data, coord=lonreg)
        mm = ax.pcolormesh(lon_cyc,\
                         latreg,\
                         data_cyc,
                         vmin = mmin,
                         vmax = mmax,
                         transform=ccrs.PlateCarree(),
                         cmap=colormap,
                        )
    else:
        raise ValueError('Inknown plot type {}'.format(ptype))

    ax.coastlines(resolution='50m', lw=0.5)
    ax.add_feature(
        cfeature.GSHHSFeature(levels=[1], scale='low', facecolor='lightgray'))
    cb = plt.colorbar(mm, orientation='horizontal', pad=0.03)
    cb.set_label(flf.variables[variable].units)
    plt.title('{} at {}m.'.format(variable, realdepth))
    plt.tight_layout()
    if ofile:
        plt.savefig(ofile, dpi=100)
    else:
        plt.show()
Exemplo n.º 5
0
def convert(meshpath, ipath, opath, variable, depths, box,
            res, influence, timestep, abg, interp, ncore, k):
    '''
    meshpath - Path to the folder with FESOM1.4 mesh files.

    ipath    - Path to FESOM1.4 netCDF file or files (with wildcard).

    opath    - path where the output will be stored.

    variable - The netCDF variable to be converted.
    '''
    print(ipath)
    mesh = pf.load_mesh(meshpath, abg=abg, usepickle=False, usejoblib=True)

    sstep = timestep
    radius_of_influence = influence

    left, right, down, up = box
    lonNumber, latNumber = res

    lonreg = np.linspace(left, right, lonNumber)
    latreg = np.linspace(down, up, latNumber)
    lonreg2, latreg2 = np.meshgrid(lonreg, latreg)

    localdir = os.path.dirname(os.path.abspath(__file__))
    # print(os.path.abspath(__file__))
    print('localdir='+localdir)
    with open(localdir+'/CMIP6_Omon.json') as data_file:
        cmore_table = json.load(data_file, object_pairs_hook=OrderedDict)

    with open(localdir+'/CMIP6_SIday.json') as data_file:
        cmore_table_ice = json.load(data_file, object_pairs_hook=OrderedDict)

    depths = np.array(depths.split(' '),dtype='float32')
    if depths[0] == -1:
        dind = range(mesh.zlevs.shape[0])
        realdepth = mesh.zlevs
    else:
        dind = []
        realdepth = []
        for depth in depths:
            ddepth = abs(mesh.zlevs-depth).argmin()
            dind.append(ddepth)
            realdepth.append(mesh.zlevs[ddepth])
    print(dind)
    print(realdepth)
    #realdepth = mesh.zlevs[dind]
    
    distances, inds = pf.create_indexes_and_distances(mesh, lonreg2, latreg2,\
                                                      k=k, n_jobs=4)

    ind_depth_all = []
    ind_noempty_all = []
    ind_empty_all = []

    for i in range(len(mesh.zlevs)):
        ind_depth, ind_noempty, ind_empty = pf.ind_for_depth(mesh.zlevs[i], mesh)
        ind_depth_all.append(ind_depth)
        ind_noempty_all.append(ind_noempty)
        ind_empty_all.append(ind_empty)
    if interp == 'nn':
        topo_interp = pf.fesom2regular(mesh.topo, mesh, lonreg2, latreg2, distances=distances,
                                inds=inds, radius_of_influence=radius_of_influence, n_jobs=1)
        k = 1
        distances, inds = pf.create_indexes_and_distances(mesh, lonreg2, latreg2,\
                                                          k=k, n_jobs=4)
        points, qh = None, None
    elif interp == 'idist':
        topo_interp = pf.fesom2regular(mesh.topo, mesh, lonreg2, latreg2, distances=distances,
                                inds=inds, radius_of_influence=radius_of_influence, n_jobs=1, how='idist')
        k = k
        distances, inds = pf.create_indexes_and_distances(mesh, lonreg2, latreg2,\
                                                          k=k, n_jobs=4)
        points, qh = None, None
    elif interp == 'linear':
        points = np.vstack((mesh.x2, mesh.y2)).T
        qh = qhull.Delaunay(points)
        topo_interp = LinearNDInterpolator(qh, mesh.topo)((lonreg2, latreg2))
        distances, inds = None, None
    elif interp == 'cubic':
        points = np.vstack((mesh.x2, mesh.y2)).T
        qh = qhull.Delaunay(points)
        topo_interp = CloughTocher2DInterpolator(qh, mesh.topo)((lonreg2, latreg2))
        distances, inds = None, None

    mdata = maskoceans(lonreg2, latreg2, topo_interp, resolution = 'h', inlands=False)
    topo = np.ma.masked_where(~mdata.mask, topo_interp)
    
    # Backend is switched to threading for linear and cubic interpolations
    # due to problems with memory mapping.
    # One have to test threading vs multiprocessing.
    if (interp == 'linear') or (interp=='cubic'):
        backend = 'threading'
    else:
        backend = 'multiprocessing'

    Parallel(n_jobs=ncore, backend=backend, verbose=50)(delayed(scalar2geo)(ifile, opath, variable,
                                       mesh, ind_noempty_all,
                                       ind_empty_all,ind_depth_all, cmore_table, lonreg2, latreg2,
                                       distances, inds, radius_of_influence, topo, points, interp, qh, timestep, dind, realdepth) for ifile in ipath)
Exemplo n.º 6
0
def showo(meshpath, ifile, variable, depth, box, timestep, minmax, mapproj,
          quiet, ofile):
    '''
    meshpath - Path to the folder with FESOM1.4 mesh files.

    ifile    - Path to FESOM1.4 netCDF file.

    variable - The netCDF variable to be plotted.
    '''
    if not quiet:
        click.secho('Mesh: {}'.format(meshpath))
        click.secho('File: {}'.format(ifile))
        click.secho('Variable: {}'.format(variable), fg='red')
        click.secho('Depth: {}'.format(depth), fg='red')
        click.secho('BOX: {}'.format(box))
        click.secho('Timestep: {}'.format(timestep))
        if minmax:
            click.secho('Min/max: {}'.format(minmax), fg='red')
        else:
            click.secho('Min/max: auto', fg='red')

    mesh = pf.load_mesh(meshpath)
    flf = Dataset(ifile)
    left, right, down, up = box

    elem_nonan, no_nan_tri = pf.cut_region(mesh, [left, right, down, up], 0)
    if variable == 'topo':
        level_data = mesh.topo
    else:
        level_data, nnn = pf.get_data(flf.variables[variable][timestep, :],
                                      mesh, depth)

    if minmax:
        mmin, mmax = minmax
    else:
        mmin = level_data[elem_nonan].min()
        mmax = level_data[elem_nonan].max()

    plt.figure(figsize=(10, 10))
    if mapproj == 'merc':
        ax = plt.subplot(111, projection=ccrs.Mercator())
    elif mapproj == 'pc':
        ax = plt.subplot(111, projection=ccrs.PlateCarree())
    elif mapproj == 'np':
        ax = plt.subplot(111, projection=ccrs.NorthPolarStereo())
    elif mapproj == 'sp':
        ax = plt.subplot(111, projection=ccrs.SouthPolarStereo())
    elif mapproj == 'rob':
        ax = plt.subplot(111, projection=ccrs.Robinson())

    ax.set_extent(box, crs=ccrs.PlateCarree())
    mm = ax.tripcolor(mesh.x2,
                      mesh.y2,
                      elem_nonan,
                      level_data,
                      transform=ccrs.PlateCarree(),
                      edgecolors='k',
                      cmap=cm.Spectral_r,
                      vmin=mmin,
                      vmax=mmax)
    ax.coastlines(lw=0.5, resolution='10m')
    plt.colorbar(
        mm,
        orientation='horizontal',
        pad=0.03,
    )
    plt.title('{} at {}'.format(variable, depth), size=20)
    if ofile:
        plt.savefig(ofile, dpi=200)
    else:
        plt.show()
    sys.argv[3]), int(sys.argv[4])

#################################################################################################
# mesh-specific configs

if meshname4plots == 'LR':

    meshpath = '/work/bm0944/input/CORE2_final/'  # COREII at DKRZ
    savepath = '/pf/a/a270046/hierarchy/new_figures/paper-animation/meridional_ATL/LR/'

    #filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output/fesom.{}.oce.mean.nc' # original COREII data at DKRZ
    filedia = '/work/ab0995/a270046/hierarchy-grids/cpl_output/fesom.{}.oce.diag.nc'  # COREII data at DKRZ
    filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output/cdo-postprocessing/fesom.{}.oce.annualmean.nc'  # annual COREII

    # load mesh
    mesh = pf.load_mesh(meshpath, get3d=True, usepickle=False, usejoblib=True)

elif meshname4plots == 'REF':

    meshpath = '/work/bm0944/input/mesh_ref87k/'  # REF at DKRZ
    savepath = '/pf/a/a270046/hierarchy/new_figures/paper-animation/meridional_ATL/REF/'

    #filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output_ref87k/fesom.{}.oce.mean.nc' # original REF data at DKRZ
    filedia = '/work/ab0995/a270046/hierarchy-grids/cpl_output_ref87k/fesom.{}.oce.diag.nc'  # REF data at DKRZ
    filetmp = '/work/ab0995/a270046/hierarchy-grids/cpl_output_ref87k/cdo-postprocessing/fesom.{}.oce.annualmean.nc'  # annual REF

    mesh = pf.load_mesh(meshpath, get3d=True, usepickle=False, usejoblib=True)

elif meshname4plots == 'MR0':

    meshpath = '/work/bm0944/input/aguv/'  # AGUV at DKRZ
Exemplo n.º 8
0
end_year = config.getint('main', 'end_year')

ifile_template = config.get('main', 'ifile_template')
ifile_template_ice = config.get('main', 'ifile_template_ice')
ofile_template = config.get('main', 'ofile_template')

distribute_timesteps = config.getboolean('main', 'distribute_timesteps')

# Generate regular grid
lon = np.linspace(left_lon, right_lon, number_of_lons)
lat = np.linspace(lower_lat, upper_lat, number_of_lats)
lons, lats = np.meshgrid(lon, lat)

# read the FESOM mesh
print('=' * 50)
mesh = pf.load_mesh(meshpath, abg=angles_for_mesh, get3d=True, usepickle=True)

# Open CMOR variable descriptions
with open('CMIP6_Omon.json') as data_file:
    cmore_table = json.load(data_file, object_pairs_hook=OrderedDict)

with open('CMIP6_SIday.json') as data_file:
    cmore_table_ice = json.load(data_file, object_pairs_hook=OrderedDict)

# Add some variables that are missing in CMOR tables
cmore_table['variable_entry']['wo'] = OrderedDict([
    (u'modeling_realm', u'ocean'), (u'standard_name', u'sea_water_z_velocity'),
    (u'units', u'm s-1'), (u'cell_methods', u'time: mean'),
    (u'cell_measures', u'--OPT'), (u'long_name', u'Sea Water Z Velocity'),
    (u'comment', u'Not standard CMORE variable'),
    (u'dimensions', u'longitude latitude olevel time'), (u'out_name', u'wo'),
Exemplo n.º 9
0
def load_mesh(mesh_path):
    """Wrapper function to load the mesh with timing"""
    return pf.load_mesh(mesh_path, usepickle=False)
Exemplo n.º 10
0
def loadmeshdata(meshpath, abg):
    mesh = pf.load_mesh(meshpath, abg=abg, usepickle=False, usejoblib=True)
    return mesh