def read(filein,latmin=None,latmax=None,lonmin=None,lonmax=None,level=None,varname=None,time=0,netcdf=None,**kwargs):
"""Return navlon,navlat,data.
"""
if netcdf=='3':
from scipy.io import netcdf
ncopen = netcdf.netcdf_file
elif netcdf=='4':
from netCDF4 import Dataset as ncopen
for key in kwargs:
exec(key + " = kwargs['" + key + "']")
ncfile = ncopen(filein,'r')
# get indices
_navlon = ncfile.variables['nav_lon'][:,:]
_navlat = ncfile.variables['nav_lat'][:,:]
if latmin is None: latmin = _navlat.min()
if latmax is None: latmax = _navlat.max()
if lonmin is None: lonmin = _navlon.min()
if lonmax is None: lonmax = _navlon.max()
domain = (lonmin<_navlon) * (_navlon<lonmax) * (latmin<_navlat) * (_navlat<latmax)
where = npy.where(domain)
vlats = _navlat[where]
vlons = _navlon[where]
jmin = where[0][vlats.argmin()]
jmax = where[0][vlats.argmax()]
imin = where[1][vlons.argmin()]
imax = where[1][vlons.argmax()]
#load arrays
navlon = _navlon[jmin:jmax+1,imin:imax+1]
navlat = _navlat[jmin:jmax+1,imin:imax+1]
if level is None:
data = ncfile.variables[varname][time,jmin:jmax+1,imin:imax+1]
else:
data = ncfile.variables[varname][time,level,jmin:jmax+1,imin:imax+1]
return navlon,navlat,data
def read_datagrid(gridfile,
latmin=None,
latmax=None,
lonmin=None,
lonmax=None):
"""Return navlon,navlat."""
ncfile = ncopen(gridfile, 'r')
# load navlon and navlat
_navlon = ncfile.variables['nav_lon'][:, :]
_navlat = ncfile.variables['nav_lat'][:, :]
#-Define domain
domain = (lonmin < _navlon) * (_navlon < lonmax) * (latmin < _navlat) * (
_navlat < latmax)
where = np.where(domain)
vlats = _navlat[where]
vlons = _navlon[where]
#get indice
jmin = where[0][vlats.argmin()]
jmax = where[0][vlats.argmax()]
imin = where[1][vlons.argmin()]
imax = where[1][vlons.argmax()]
#load arrays
navlon = _navlon[jmin:jmax + 1, imin:imax + 1]
navlat = _navlat[jmin:jmax + 1, imin:imax + 1]
return navlon, navlat, jmin, jmax, imin, imax
def nc_get_dim(ncfil,vblname):
"""
vbl=nc_get_var(ncfil,vblname,time_index=None)
*ncfil is string (filename)
*vname is string (variable name)
*vbl is a mod_reading.var_object instance
"""
nc = ncopen(ncfil)
if vblname in nc.variables:
vbl0 = nc.variables[vblname]
# get the netcdf attributes
attlist = vbl0.ncattrs()
attvals = []
for att in attlist:
attval = getattr(vbl0,att)
attvals.append(attval)
Xatts = [attlist,attvals]
vals = vbl0[:]
nc.close()
return MR.var_object(vals, extra_atts=Xatts)
else:
raise ValueError(vbl_name+' not given as an array')
def read(filein,latmin=None,latmax=None,lonmin=None,lonmax=None,level=None,varname=None,time=0,netcdf=None,**kwargs):
"""Return navlon,navlat,data.
"""
if netcdf=='3':
from scipy.io import netcdf
ncopen = netcdf.netcdf_file
elif netcdf=='4':
from netCDF4 import Dataset as ncopen
for key in kwargs:
exec(key + " = kwargs['" + key + "']")
ncfile = ncopen(filein,'r')
# get indices
_navlon = ncfile.variables['nav_lon'][:,:]
_navlat = ncfile.variables['nav_lat'][:,:]
if latmin is None: latmin = _navlat.min()
if latmax is None: latmax = _navlat.max()
if lonmin is None: lonmin = _navlon.min()
if lonmax is None: lonmax = _navlon.max()
domain = (lonmin<_navlon) * (_navlon<lonmax) * (latmin<_navlat) * (_navlat<latmax)
where = npy.where(domain)
vlats = _navlat[where]
vlons = _navlon[where]
jmin = where[0][vlats.argmin()]
jmax = where[0][vlats.argmax()]
imin = where[1][vlons.argmin()]
imax = where[1][vlons.argmax()]
#load arrays
navlon = _navlon[jmin:jmax+1,imin:imax+1]
navlat = _navlat[jmin:jmax+1,imin:imax+1]
if level is None:
data = ncfile.variables[varname][time,jmin:jmax+1,imin:imax+1]
else:
data = ncfile.variables[varname][time,level,jmin:jmax+1,imin:imax+1]
return navlon,navlat,data
def lonlat_names(ncfil):
nc = ncopen(ncfil)
for vbl in nc.variables:
if 'lon' in vbl[:3].lower():
lonname = vbl
if 'lat' in vbl[:3].lower():
latname = vbl
nc.close()
return lonname, latname
def nc_get_time(ncfil, time_name='time'):
"""
vbl=nc_get_time(ncfil, time_name)
*ncfil is string (filename)
*vname is string (variable name)
*time_index is record number to get
*vbl is a mod_reading.var_object instance
"""
with ncopen(ncfil) as nc:
time = nc.variables[time_name][:]
return time
def nc_get_var(ncfil, vblname, time_index=None):
#get basic info from 2d variable
########################################################
class def_vbl:
#####################################################
def __init__(self, vbl0, time_index=None):
# get the netcdf attributes
ncatts = vbl0.ncattrs()
for att in ncatts:
attval = getattr(vbl0, att)
setattr(self, att, attval)
if hasattr(vbl0, 'dimensions'):
dims = vbl0.dimensions
self.dimensions = dims
##################################################
# some attributes that depend on rank
if vbl0.ndim == 1:
vals = vbl0[:]
elif vbl0.ndim == 2:
vals = vbl0[:, :]
elif vbl0.ndim == 3:
if time_index is None:
vals = vbl0[:, :, :]
else:
vals = vbl0[time_index, :, :]
if hasattr(vbl0, 'dimensions'):
self.dimensions = dims[1:] # drop time dimension
##################################################
self.values = vals
self.shape = vals.shape
self.ndim = vals.ndim
self.__getitem__ = self.values.__getitem__
#####################################################
########################################################
nc = ncopen(ncfil)
vbl0 = nc.variables[vblname]
vbl = def_vbl(vbl0, time_index)
nc.close()
return vbl
def nc_get_var_atts(ncfil,vblname):
"""
vbl=nc_get_var_atts(ncfil, vblname, time_index=None)
Parameters:
*ncfil is string (filename)
*vname is string (variable name)
Returns:
*vbl is a mod_reading.var_object instance
"""
nc = ncopen(ncfil)
vbl0 = nc.variables[vblname]
# get the netcdf attributes
attlist = vbl0.ncattrs()
atts = {}
for att in attlist:
attval = getattr(vbl0,att)
atts.update({att:attval})
nc.close()
return atts
def get_lonlat(self, vec2mat=True, **kwargs):
"""
Parameters:
* vec2mat (bool): if lon, lat are vectors (eg if they are dimensions),
convert to matrices
* ij_range: list of integers to reduce the size of arrays
- can be: None or [i0, i1, j0, j1]
- let lon_all, lat_all = self.get_lonlat(vec2mat=True)
- let lon, lat = self.get_lonlat(vec2mat=True, ij_range=[i0, i1, j0, j1])
- lon = lon_all[i0:i1, j0:j1], lat = lat_all[i0:i1, j0:j1]
Returns:
* lon, lat (np.array)
"""
if self.timedep_lonlat:
# return lon,lat using get_var
lon = self.get_var(self.lonname, **kwargs)
lat = self.get_var(self.latname, **kwargs)
return (lon.values.filled(np.nan),
lat.values.filled(np.nan))
ij_range = kwargs.get('ij_range', None)
with ncopen(self.lonlat_file) as nc:
lono = nc.variables[self.lonname]
lato = nc.variables[self.latname]
if ij_range is None:
if lono.ndim==2:
lon = lono[:, :]
lat = lato[:, :]
else:
lon = lono[:]
lat = lato[:]
if vec2mat:
if self.lon_first:
# lon in cols, lat in rows
lon, lat = np.meshgrid(lon, lat, indexing='ij')
else:
# lon in rows, lat in cols
lon, lat = np.meshgrid(lon, lat, indexing='xy')
else:
print('ij_range: ', ij_range)
i0, i1, j0, j1 = ij_range
if lono.ndim==2:
lon = lono[i0:i1, j0:j1]
lat = lato[i0:i1, j0:j1]
else:
if self.lon_first:
# lon in cols, lat in rows
lon = lono[j0:j1]
lat = lato[i0:i1]
else:
# lon in rows, lat in cols
lon = lono[i0:i1]
lat = lato[j0:j1]
if vec2mat:
if self.lon_first:
# lon in cols, lat in rows
lon, lat = np.meshgrid(lon, lat, indexing='ij')
else:
# lon in rows, lat in cols
lon, lat = np.meshgrid(lon, lat, indexing='xy')
return lon, lat
def __init__(self,
ncfil,
time_index=None,
lonlat_file=None,
timedep_lonlat=False):
# NB time_index not needed here, but adding it as a dummy index
# means this object can be used along with other similar objects
# in some functions in mod_reading.py
self.filename = ncfil
if ncfil[0]=='/':
bn = os.path.basename(ncfil)
self.basedir = ncfil.strip(bn)
else:
bn = ncfil
self.basedir = os.getcwd()+'/'
self.basename = os.path.splitext(bn)[0]
self.filetype = 'netcdf'
self.object_type = 'netcdf'
# things to work with plotting stuff in mod_reading.py
self.HYCOM_region = None
self.get_fixed_lonlat = self.get_lonlat
# for drifters
self.timedep_lonlat = timedep_lonlat
# added here manually
# - TODO could possibly be determined
# from netcdf metadata though
# - could also be an input
self.reftime_sig = 'start of forecast'
# open the file
nc = ncopen(ncfil)
dkeys = list(nc.dimensions.keys())
vkeys = list(nc.variables.keys())
# remove dimensions from variables
self.dimensions = dkeys
for key in dkeys:
if key in vkeys:
vkeys.remove(key)
Nkeys = len(vkeys)
# time info:
self._set_time_info(nc)
# get global netcdf attributes
class ncatts:
def __init__(self,nc):
for att in nc.ncattrs():
attval = getattr(nc,att)
setattr(self,att,attval)
return
def atts2list(self):
return vars(self).keys()
self.ncattrs = ncatts(nc)
# grid info:
if lonlat_file is None:
lonlat_file = ncfil
self.lonlat_file = lonlat_file
# are lon,lat dimensions?
self.lonname, self.latname = lonlat_names(self.lonlat_file)
self.lonlat_dim = (self.lonname in self.dimensions)
# basic lon-lat info
with ncopen(self.lonlat_file) as nc2:
lon = nc2.variables[self.lonname]
lat = nc2.variables[self.latname]
if self.lonlat_dim:
self.lon0 = lon[0]
self.lat0 = lat[0]
self.lonlat_corners = np.array([
(lon[0], lat[0]),
(lon[-1], lat[0]),
(lon[-1], lat[-1]),
(lon[0], lat[-1]),
(lon[0], lat[0]),
]).T
# get example variable:
# - for eg plotting, need to make the lon/lat matrices
# (converted from vectors)
# have the same shape as the variables
for vkey in vkeys:
if self.lonname in nc.variables[vkey].dimensions:
ncvar = nc.variables[vkey]
break
for dkey in ncvar.dimensions:
if dkey==self.lonname:
self.lon_first = True
self.shape = (len(lon), len(lat))
break
elif dkey==self.latname:
self.lon_first = False
self.shape = (len(lat), len(lon))
break
elif self.timedep_lonlat:
self.lon0 = None
self.lat0 = None
self.lonlat_corners = None
self.shape = lon[0,:,:].shape
else:
self.lon0 = lon[0, 0]
self.lat0 = lat[0, 0]
self.lonlat_corners = np.array([
(lon[0, 0], lat[0, 0]),
(lon[-1, 0], lat[-1, 0]),
(lon[-1, -1], lat[-1, -1]),
(lon[0, -1], lat[0, -1]),
(lon[0, 0], lat[0, 0]),
]).T
self.lon1 = lon[-1, -1]
self.lat1 = lat[-1, -1]
self.shape = lon.shape
ny, nx = self.shape
self.Npts_x = nx # No of points in x dirn
self.Npts_y = ny # No of points in y dirn
self.Npts = nx*ny # Total no of points
# projection info:
proj_list = [
'stereographic',
'projection_3',
'Polar_Stereographic_Grid',
'Lambert_Azimuthal_Grid',
]
# could also have mercator or regular lon-lat
HAVE_PROJ = 0 # if 0 assume HYCOM native grid
keys_to_remove = []
for proj_name in proj_list:
if proj_name in vkeys:
proj = nc.variables[proj_name]
att_list = proj.ncattrs()
HAVE_PROJ = 1
keys_to_remove.append(proj_name) # don't want to keep projection variable with the other variables
break
if HAVE_PROJ:
# object with the netcdf attributes of projection variable
# + some extra proj-dependent info
att_list_full = [att_list[i] for i in range(len(att_list))]
att_vals_full = []
for att in att_list:
att_val = proj.getncattr(att)
att_vals_full.append(att_val)
self.proj_info = MR.proj_obj(att_list_full, att_vals_full)
else:
self.proj_info = None
# variable list
# - remove some other variables from vkeys
# - eg projection,lon,lat
# - time_bnds
# - TODO model_depth?
keys_to_remove.append("time_bnds")
# keys_to_remove.append('model_depth')
if not self.timedep_lonlat:
keys_to_remove.extend([self.lonname, self.latname])
# other variables to remove
for key in keys_to_remove:
if key in vkeys:
vkeys.remove(key)
self.variables = vkeys
self.variables3d = None #TODO enable treatment of 3d fields
self.all_variables = vkeys
nc.close()
return
def nc_get_var(ncfil, vblname, time_index=None,
depth_index=0, ij_range=None, **kwargs):
"""
vbl=nc_get_var(ncfil, vblname, time_index=None)
*ncfil is string (filename)
*vname is string (variable name)
*time_index is record number to get
*depth_index is horizon number to get
*vbl is a mod_reading.var_object instance
"""
# NB kwargs is not used, but is there as a dummy to avoid having to sort kwargs
# before calling this function
with ncopen(ncfil) as nc:
vbl0 = nc.variables[vblname]
# get the netcdf attributes
attlist = vbl0.ncattrs()
attvals = []
for att in attlist:
attval = getattr(vbl0, att)
attvals.append(attval)
dims = vbl0.dimensions
shape = vbl0.shape
# do we want to limit the range
if ij_range is not None:
i0, i1, j0, j1 = ij_range
# some attributes that depend on rank
if vbl0.ndim==1:
vals = vbl0[:]
elif vbl0.ndim==2:
if ij_range is not None:
vals = vbl0[i0:i1, j0:j1]
else:
vals = vbl0[:, :]
elif vbl0.ndim==3:
if time_index is None:
if shape[0]==1:
time_index = 0
if time_index is None:
if ij_range is not None:
vals = vbl0[:, i0:i1, j0:j1]
else:
vals = vbl0[:, :, :]
else:
if ij_range is not None:
vals = vbl0[time_index, i0:i1, j0:j1]
else:
vals = vbl0[time_index, :, :]
dims = dims[1:]
elif vbl0.ndim==4:
if time_index is None:
if shape[0]==1:
time_index = 0
if time_index is None:
if ij_range is not None:
vals = vbl0[:, depth_index, i0:i1, j0:j1]
else:
vals = vbl0[:, depth_index, :, :]
dims = (dims[0], dims[2], dims[3])
else:
if ij_range is not None:
vals = vbl0[time_index, depth_index, i0:i1, j0:j1]
else:
vals = vbl0[time_index, depth_index, :, :]
dims = dims[2:]
attlist.append('dimensions')
attvals.append(dims)
return MR.var_object(vals, extra_atts=[attlist, attvals])
def nc_getinfo(ncfil, time_index=None):
########################################################
class nc_info:
#####################################################
def __init__(self):
self.ncattrs = 0 # global netcdf attributes
self.proj_info = 0 # info about projection
self.variable_list = 0 # list of variable names
self.lon0 = 0 # 1st longitude value
self.lat0 = 0 # 1st latitude value
self.shape = 0 # shape of grid
self.Npts_x = 0 # No of points in x dirn
self.Npts_y = 0 # No of points in y dirn
self.Npts = 0 # Total no of points
self.reftime = 0
# added here manually
# - could possibly be determined
# from netcdf metadata though
self.reftime_sig = 'start of forecast'
self.timevalues = []
self.timeunits = 'hour'
self.datatime = 0
self.number_of_time_records = 1
########################################################
ncinfo = nc_info()
nc = ncopen(ncfil)
# get global netcdf attributes
ncinfo.ncattrs = nc.ncattrs()
dkeys = nc.dimensions.keys()
vkeys = nc.variables.keys()
Nkeys = len(vkeys)
########################################################
# time info:
time = nc.variables['time']
Nt = len(time[:])
reftime_u = time[0] # hours since refpoint
time_info = time.units.split()
#
time_info[0] = time_info[0].strip('s') # 1st word gives units
if time_info[0] == 'econd':
time_info[0] = 'second'
#
tu = time.units
lst = tu.split()
date2 = lst[2]
# i0 = tu.index('-')
# date0 = tu[i0-4:i0+6]
# i1 = tu.index(':')
# date1 = tu[i1-2:i1+6]
# #
# date2 = date0+date1+'Z'
# time_fmt = '%Y-%m-%dT0%H:%M:%SZ' # eg 1950-1-1T12:00:00Z
time_fmt = '%Y-%m-%dT%H:%M:%SZ' # eg 1950-1-1T12:00:00Z
refpoint = datetime.strptime(date2, time_fmt)
#
if time_info[0] == 'second':
ncinfo.reftime = refpoint + timedelta(seconds=reftime_u)
elif time_info[0] == 'hour':
ncinfo.reftime = refpoint + timedelta(hours=reftime_u)
if time_info[0] == 'hour':
ncinfo.timeunits = time_info[0]
ncinfo.timevalues = [time[i] - time[0] for i in range(Nt)]
elif time_info[0] == 'second':
# convert time units to hours for readability of the messages:
ncinfo.timeunits = 'hour'
ncinfo.timevalues = [
int((time[i] - time[0]) / 3600.) for i in range(Nt)
]
if time_index is not None:
ncinfo.datatime = ncinfo.timevalues[time_index]
ncinfo.number_of_time_records = Nt
########################################################
########################################################
# grid info:
ncinfo.lon0 = nc.variables['longitude'][0, 0]
ncinfo.lat0 = nc.variables['latitude'][0, 0]
ncinfo.shape = nc.variables['latitude'][:, :].shape
ny, nx = ncinfo.shape
ncinfo.Npts_x = nx # No of points in x dirn
ncinfo.Npts_y = ny # No of points in y dirn
ncinfo.Npts = nx * ny # Total no of points
ncinfo.shape = nc.variables['latitude'][:, :].shape
########################################################
########################################################
# projection info:
proj_list = ['stereographic',
'projection_3'] # could also have mercator or regular lon-lat
for proj_name in proj_list:
if proj_name in vkeys:
proj = nc.variables[proj_name]
att_list = proj.ncattrs()
break
# object with the netcdf attributes of projection variable
# + some extra proj-dependent info
att_list_full = [att_list[i] for i in range(len(att_list))]
if proj_name == 'stereographic':
att_list_full.extend(['x_resolution', 'y_resolution'])
ncinfo.proj_info = proj_obj(att_list_full)
for att in att_list:
attval = proj.getncattr(att)
setattr(ncinfo.proj_info, att, attval)
if proj_name == 'stereographic':
# add x,y resolution to ncinfo.proj_info
xx = nc.variables['x'][0:2]
yy = nc.variables['y'][0:2]
dx = xx[1] - xx[0] # units of 100km
dy = yy[1] - yy[0] # units of 100km
fac = 1e5 # convert from 100km to m
#
ncinfo.proj_info.x_resolution = dx * fac
ncinfo.proj_info.y_resolution = dy * fac
########################################################
########################################################
# variable list
vlist = []
bkeys = [proj_name, 'longitude', 'latitude', 'model_depth']
bkeys.extend(dkeys)
for key in vkeys:
if key not in bkeys:
vlist.append(key)
ncinfo.variable_list = vlist
########################################################
nc.close()
return ncinfo
from netCDF4 import Dataset as ncopen
import numpy as np
from datetime import datetime
"""
fake data to be written
"""
Ntot = 100
uice = 1. + np.zeros(Ntot)
vice = np.zeros(Ntot)
cno = np.zeros(Ntot, dtype='int')
cno[50:] = 1
nc = ncopen('sample.nc', 'w', format='NETCDF4') #'w' stands for write
"""
The above line creates a netCDF file called "sample.nc" in the current folder. nc is a netCDF Dataset object that provides methods for storing data to the
file. nc also doubles as the root group. A netCDF group is basically a directory or folder within the netCDF dataset. This allows you to organize data as you would in a unix file system. Let's create a group for the heck of it:
"""
if 0:
tempgrp = nc.createGroup('Temp_data')
else:
tempgrp = nc
"""
Specifying dimensions
The next step is to specify the dimensions of the data. If you plan to save a multidimensional array of data, each dimension of that array needs to be given a name and a length:
"""
tempgrp.createDimension('point_index', Ntot)
tempgrp.createDimension('time', 1)
"""
Building variables
for D in Vcont:
N = D['length']
C = D['contour_value']
lonc = D['lon']
latc = D['lat']
cno_vec[position:position + N] = contour_number
c_vec[position:position + N] = C
lon_vec[position:position + N] = lonc
lat_vec[position:position + N] = latc
#increment position
position += N
contour_number += 1
print('Writing output netcdf file...')
nc = ncopen(ncfil2, 'w', format='NETCDF4') #'w' stands for write
"""
The above line creates a netCDF file called "sample.nc" in the current folder. nc is a netCDF Dataset object that provides methods for storing data to the
file.
"""
if 0:
"""
nc also doubles as the root group. A netCDF group is basically a directory or folder within the netCDF dataset.
This allows you to organize data as you would in a unix file system. Let's create a group for the heck of it:
"""
tempgrp = nc.createGroup('Temp_data')
else:
# write directly to the root group
tempgrp = nc
"""
