def wrapper(*args, **kws):
def _mod(n):
k = kws
s = shared
return (n in s) and ((n not in k) or (n in k and k[n] is None))
if 'filename' in kws and kws['filename'] is not None:
kws['data_file'] = kws['grid_file'] = kws['filename']
ds = dg = None
if _mod('dataset'):
if 'grid_file' in kws and 'data_file' in kws:
if kws['grid_file'] == kws['data_file']:
ds = dg = get_dataset(kws['grid_file'])
else:
ds = get_dataset(kws['data_file'])
dg = get_dataset(kws['grid_file'])
kws['dataset'] = ds
else:
if 'grid_file' in kws and kws['grid_file'] is not None:
dg = get_dataset(kws['grid_file'])
else:
dg = kws['dataset']
ds = kws['dataset']
if _mod('grid'):
gt = kws.get('grid_topology', None)
kws['grid'] = Grid.from_netCDF(kws['filename'], dataset=dg, grid_topology=gt)
if kws.get('varnames', None) is None:
varnames = cls._gen_varnames(kws['data_file'],
dataset=ds)
# if _mod('time'):
# time = Time.from_netCDF(filename=kws['data_file'],
# dataset=ds,
# varname=data)
# kws['time'] = time
return func(*args, **kws)
def _find_required_grid_attrs(cls,
filename,
dataset=None,
grid_topology=None):
# THESE ARE ACTUALLY ALL OPTIONAL. This should be migrated when optional attributes
# are dealt with
# Get superset attributes
gf_vars = dataset.variables if dataset is not None else get_dataset(
filename).variables
gf_vars = dict([(k.lower(), v) for k, v in gf_vars.items()])
init_args, gt = super(Grid_R, cls)._find_required_grid_attrs(
filename, dataset=dataset, grid_topology=grid_topology)
# Grid_R only needs node_lon and node_lat. However, they must be a specific shape (1D)
node_lon = init_args['node_lon']
node_lat = init_args['node_lat']
if len(node_lon.shape) != 1:
raise ValueError(
'Too many dimensions in node_lon. Must be 1D, was {0}D'.format(
len(node_lon.shape)))
if len(node_lat.shape) != 1:
raise ValueError(
'Too many dimensions in node_lat. Must be 1D, was {0}D'.format(
len(node_lat.shape)))
return init_args, gt
def _find_required_grid_attrs(cls, filename, dataset=None, grid_topology=None):
# THESE ARE ACTUALLY ALL OPTIONAL. This should be migrated when optional attributes
# are dealt with
# Get superset attributes
gf_vars = dataset.variables if dataset is not None else get_dataset(filename).variables
gf_vars = dict([(k.lower(), v) for k, v in gf_vars.items()])
init_args, gt = super(Grid_S, cls)._find_required_grid_attrs(filename,
dataset=dataset,
grid_topology=grid_topology)
center_attrs = ['center_lon', 'center_lat']
edge1_attrs = ['edge1_lon', 'edge1_lat']
edge2_attrs = ['edge2_lon', 'edge2_lat']
node_mask = 'node_mask'
center_mask = 'center_mask'
edge1_mask = 'edge1_mask'
edge2_mask = 'edge2_mask'
center_coord_names = [['center_lon', 'center_lat'], ['lon_rho', 'lat_rho'], ['lonc', 'latc']]
edge1_coord_names = [['edge1_lon', 'edge1_lat'], ['lon_u', 'lat_u']]
edge2_coord_names = [['edge2_lon', 'edge2_lat'], ['lon_v', 'lat_v']]
node_mask_names = ['mask_psi']
center_mask_names = ['mask_rho']
edge1_mask_names = ['mask_u']
edge2_mask_names = ['mask_v']
if grid_topology is None:
for attr, names, maskattr, maskname in (zip((center_attrs, edge1_attrs, edge2_attrs),
(center_coord_names, edge1_coord_names, edge2_coord_names),
(center_mask, edge1_mask, edge2_mask),
(center_mask_names, edge1_mask_names, edge2_mask_names))):
for n1, n2 in names:
if n1 in gf_vars and n2 in gf_vars:
mask = False
#for n in maskname:
#if n in gf_vars:
#mask = gen_mask(gf_vars[n])
a1 = gf_vars[n1][:]
a2 = gf_vars[n2][:]
init_args[attr[0]] = a1
init_args[attr[1]] = a2
if maskname[0] in gf_vars:
init_args[maskattr] = gf_vars[maskname[0]]
gt[maskattr] = maskname[0]
gt[attr[0]] = n1
gt[attr[1]] = n2
break
if 'node_lon' in init_args and 'node_lat' in init_args:
mask = False
for name in node_mask_names:
if name in gf_vars:
init_args[node_mask] = gf_vars[name]
gt[node_mask] = name
else:
for n, v in grid_topology.items():
if n in center_attrs + edge1_attrs + edge2_attrs and v in gf_vars:
init_args[n] = gf_vars[v][:]
return init_args, gt
def __init__(
self,
ncfile=None,
grid=None,
variables=None,
grid_topology=None,
):
"""
Construct a gridded.Dataset object. Can be constructed from a data file,
or also raw grid and variable objects.
:param ncfile: a file to load the Dataset from.
:type ncfile: filename of netcdf file or opendap url or open netCDF4 Dataset object
(could be other file types in the future)
:param grid: a dataset.Grid object or anything that presents the same API.
:param variables: a dict of dataset.Variable objects -- or anything that
presents the same API.
Either a filename or grid and variable objects should be provided -- not both.
"""
if ncfile is not None:
self.nc_dataset = get_dataset(ncfile)
self.filename = self.nc_dataset.filepath()
# self.grid = pyugrid.UGrid.from_nc_dataset(ds)
self.grid = Grid.from_netCDF(filename=self.filename,
dataset=self.nc_dataset)
# var_names = pyugrid.read_netcdf.find_variables(self.nc_dataset,
# self.grid.mesh_name)
self.variables = self._load_variables(self.nc_dataset)
else: # no file passed in -- create from grid and variables
self.filename = None
self.grid = grid
self.variables = variables
def _find_required_grid_attrs(cls, filename, dataset=None, grid_topology=None):
# THESE ARE ACTUALLY ALL OPTIONAL. This should be migrated when optional attributes are dealt with
# Get superset attributes
gf_vars = dataset.variables if dataset is not None else get_dataset(filename).variables
gf_vars = dict([(k.lower(), v) for k, v in gf_vars.items()] )
init_args, gt = super(Grid_S, cls)._find_required_grid_attrs(filename,
dataset=dataset,
grid_topology=grid_topology)
center_attrs = ['center_lon', 'center_lat']
edge1_attrs = ['edge1_lon', 'edge1_lat']
edge2_attrs = ['edge2_lon', 'edge2_lat']
center_coord_names = [['center_lon', 'center_lat'], ['lon_rho', 'lat_rho'], ['lonc', 'latc']]
edge1_coord_names = [['edge1_lon', 'edge1_lat'], ['lon_u', 'lat_u']]
edge2_coord_names = [['edge2_lon', 'edge2_lat'], ['lon_v', 'lat_v']]
if grid_topology is None:
for attr, names in (zip((center_attrs, edge1_attrs, edge2_attrs),
(center_coord_names, edge1_coord_names, edge2_coord_names))):
for n1, n2 in names:
if n1 in gf_vars and n2 in gf_vars:
init_args[attr[0]] = gf_vars[n1][:]
init_args[attr[1]] = gf_vars[n2][:]
gt[attr[0]] = n1
gt[attr[1]] = n2
break
else:
for n, v in grid_topology.items():
if n in center_attrs + edge1_attrs + edge2_attrs and v in gf_vars:
init_args[n] = gf_vars[v][:]
return init_args, gt
def from_netCDF(filename=None,
dataset=None,
depth_type=None,
varname=None,
topology=None,
_default_types=(('level', L_Depth), ('sigma', S_Depth),
('surface', DepthBase)),
*args,
**kwargs):
'''
:param filename: File containing a depth
:param dataset: Takes precedence over filename, if provided.
:param depth_type: Must be provided if autodetection is not possible.
:returns: Instance of L_Depth or S_Depth
'''
df = dataset if filename is None else get_dataset(filename, dataset)
if df is None:
raise ValueError('No filename or dataset provided')
cls = depth_type
if (depth_type is None or isinstance(depth_type, string_types)
or not issubclass(depth_type, DepthBase)):
cls = Depth._get_depth_type(df, depth_type, topology,
_default_types)
return cls.from_netCDF(filename=filename,
dataset=dataset,
topology=topology,
**kwargs)
def _gen_varname(cls,
filename=None,
dataset=None,
names_list=None,
std_names_list=None):
"""
Function to find the default variable names if they are not provided.
:param filename: Name of file that will be searched for variables
:param dataset: Existing instance of a netCDF4.Dataset
:type filename: string
:type dataset: netCDF.Dataset
:return: List of default variable names, or None if none are found
"""
df = None
if dataset is not None:
df = dataset
else:
df = get_dataset(filename)
if names_list is None:
names_list = cls.default_names
if std_names_list is None:
std_names_list = cls.cf_names
for n in names_list:
if n in df.variables.keys():
return n
for n in std_names_list:
for var in df.variables.values():
if (hasattr(var, 'standard_name') and var.standard_name == n
or hasattr(var, 'long_name') and var.long_name == n):
return var.name
raise ValueError("Default names not found.")
def from_netCDF(cls,
filename=None,
dataset=None,
name=None,
topology=None,
terms=None,
surface_index=None,
bottom_index=None,
**kwargs):
df = dataset if filename is None else get_dataset(filename, dataset)
if df is None:
raise ValueError('No filename or dataset provided')
if name is None:
name = cls.__name__ + str(cls._def_count)
if terms is None:
terms = {}
for tn, tln in cls.default_terms:
vname = tn
if tn not in dataset.variables.keys():
vname = cls._gen_varname(filename, dataset, [tn], [tln])
terms[tn] = dataset[vname][:]
if surface_index is None:
surface_index = np.argmin(terms['depth_levels'])
if bottom_index is None:
bottom_index = np.argmax(terms['depth_levels'])
return cls(name=name,
terms=terms,
surface_index=surface_index,
bottom_index=bottom_index,
**kwargs)
def _find_required_grid_attrs(cls, filename, dataset=None, grid_topology=None):
gf_vars = dataset.variables if dataset is not None else get_dataset(filename).variables
gf_vars = dict([(k.lower(), v) for k, v in gf_vars.items()])
# Get superset attributes
init_args, gt = super(Grid_U, cls)._find_required_grid_attrs(filename=filename,
dataset=dataset,
grid_topology=grid_topology)
face_attrs = ['faces']
face_var_names = ['faces', 'tris', 'nv', 'ele']
if grid_topology is None:
for n in face_var_names:
if n in gf_vars:
init_args[face_attrs[0]] = gf_vars[n][:]
gt[face_attrs[0]] = n
break
if face_attrs[0] not in init_args:
raise ValueError('Unable to find face connectivity array.')
else:
for n, v in grid_topology.items():
if n in face_attrs:
init_args[n] = gf_vars[v][:]
break
if init_args['faces'].shape[0] == 3:
init_args['faces'] = np.ascontiguousarray(np.array(init_args['faces']).T - 1)
return init_args, gt
def __init__(self,
ncfile=None,
grid=None,
variables=None,
grid_topology=None,
attributes=None):
"""
Construct a gridded.Dataset object. Can be constructed from a data file,
or also raw grid and variable objects.
:param ncfile: A file or files to load the Dataset from.
:type ncfile: Can be one of:
- file path of netcdf file as a string
- opendap url
- list of file paths (uses a netCDF4 MFDataset)
- open netCDF4 Dataset object
(could be other file types in the future)
:param grid: a dataset.Grid object or anything that presents the same API.
:param variables: a dict
of dataset.Variable objects -- or anything that
presents the same API.
:param grid_topology: mapping of grid topology components to netcdf variable names.
used to load non-confirming files.
:type grid_topology: mapping with keys of topology components and values are
variable names.
:param attributes: The global attributes of the dataset -- usually the global
attributes of a netcdf file.
:type attributes: Mapping of attribute name to attributes themselves
(usually strings)
Either a filename or grid and variable objects should be provided -- not both.
If a filename is passed in, the attributes will be pulled from the file, and
the input ones ignored.
"""
if ncfile is not None:
if (grid is not None or variables is not None
or attributes is not None):
raise ValueError(
"You can create a Dataset from a file, or from raw data"
"but not both.")
self.nc_dataset = get_dataset(ncfile)
self.filename = self.nc_dataset.filepath()
self.grid = Grid.from_netCDF(filename=self.filename,
dataset=self.nc_dataset,
grid_topology=grid_topology)
self.variables = self._load_variables(self.nc_dataset)
self.attributes = get_dataset_attrs(self.nc_dataset)
else: # no file passed in -- create from grid and variables
self.filename = None
self.grid = grid
self.variables = variables
self.attributes = {} if attributes is None else attributes
def _find_required_depth_attrs(cls, filename, dataset=None, topology=None):
# THESE ARE ACTUALLY ALL OPTIONAL. This should be migrated when optional attributes are dealt with
# Get superset attributes
gf_vars = dataset.variables if dataset is not None else get_dataset(
filename).variables
gf_vars = dict([(k.lower(), v) for k, v in gf_vars.items()])
init_args, gt = super(L_Depth, cls)._find_required_depth_attrs(
filename, dataset=dataset, topology=topology)
return init_args, gt
def _find_topology_var(filename, dataset=None):
gf = get_dataset(filename, dataset)
gts = []
for k, v in gf.variables.items():
if hasattr(v, 'cf_role') and 'topology' in v.cf_role:
gts.append(v)
# gts = gf.get_variables_by_attributes(cf_role=lambda t: t is not None and 'topology' in t)
if len(gts) != 0:
return gts[0]
else:
return None
def _find_required_grid_attrs(
cls,
filename,
dataset=None,
grid_topology=None,
):
'''
This function is the top level 'search for attributes' function. If there are any
common attributes to all potential grid types, they will be sought here.
This function returns a dict, which maps an attribute name to a netCDF4
Variable or numpy array object extracted from the dataset. When called from
Grid_U or Grid_S, this function should provide all the kwargs needed to
create a valid instance.
'''
gf_vars = dataset.variables if dataset is not None else get_dataset(
filename).variables
gf_vars = dict([(k.lower(), v) for k, v in gf_vars.items()])
init_args = {}
gt = {}
init_args['filename'] = filename
node_attrs = ['node_lon', 'node_lat']
node_coord_names = [['node_lon', 'node_lat'], ['lon', 'lat'],
['lon_psi', 'lat_psi'], ['longitude', 'latitude']]
composite_node_names = ['nodes', 'node']
if grid_topology is None:
for n1, n2 in node_coord_names:
if n1 in gf_vars and n2 in gf_vars:
init_args[node_attrs[0]] = gf_vars[n1][:]
init_args[node_attrs[1]] = gf_vars[n2][:]
gt[node_attrs[0]] = n1
gt[node_attrs[1]] = n2
break
if node_attrs[0] not in init_args:
for n in composite_node_names:
if n in gf_vars:
v = gf_vars[n][:].reshape(-1, 2)
init_args[node_attrs[0]] = v[:, 0]
init_args[node_attrs[1]] = v[:, 1]
gt['node_coordinates'] = n
break
if node_attrs[0] not in init_args:
raise ValueError('Unable to find node coordinates.')
else:
for n, v in grid_topology.items():
if n in node_attrs:
init_args[n] = gf_vars[v][:]
if n in composite_node_names:
v = gf_vars[n][:].reshape(-1, 2)
init_args[node_attrs[0]] = v[:, 0]
init_args[node_attrs[1]] = v[:, 1]
return init_args, gt
def from_netCDF(filename=None,
dataset=None,
grid_type=None,
grid_topology=None,
_default_types=(('ugrid', Grid_U),
('sgrid', Grid_S),
('rgrid', Grid_R)),
*args,
**kwargs):
'''
:param filename: File containing a grid
:param dataset: Takes precedence over filename, if provided.
:param grid_type: Must be provided if Dataset does not have a 'grid_type' attribute,
or valid topology variable
:param grid_topology: A dictionary mapping of grid attribute to variable name.
Takes precedence over discovered attributes
:param kwargs: All kwargs to SGrid, UGrid, or RGrid are valid, and take precedence
over all.
:returns: Instance of Grid_U, Grid_S, or Grid_R
'''
gf = dataset if filename is None else get_dataset(filename, dataset)
if gf is None:
raise ValueError('No filename or dataset provided')
cls = grid_type
if (grid_type is None or
isinstance(grid_type, string_types) or
not issubclass(grid_type, GridBase)):
cls = Grid._get_grid_type(gf, grid_type, grid_topology, _default_types)
# if grid_topology is passed in, don't look for the variable
if not grid_topology:
compliant = Grid._find_topology_var(None, gf)
else:
compliant = None
if compliant is not None:
c = Grid._load_grid(filename, cls, dataset)
c.grid_topology = compliant.__dict__
else:
init_args, gt = cls._find_required_grid_attrs(filename,
dataset=dataset,
grid_topology=grid_topology)
c = cls(**init_args)
c.grid_topology = gt
return c
def _load_grid(filename, grid_type, dataset=None):
'''
Redirect to grid-specific loading routine.
'''
if issubclass(grid_type, UGrid):
return grid_type.from_ncfile(filename)
elif issubclass(grid_type, SGrid):
ds = get_dataset(filename, dataset)
g = grid_type.load_grid(ds)
g.filename = filename
return g
else:
return grid_type.from_ncfile(filename)
pass
def _find_required_depth_attrs(cls, filename, dataset=None, depth_topology=None):
'''
This function is the top level 'search for attributes' function. If there are any
common attributes to all potential depth types, they will be sought here.
This function returns a dict, which maps an attribute name to a netCDF4
Variable or numpy array object extracted from the dataset. When called from
a child depth object, this function should provide all the kwargs needed to
create a valid instance using the __init__.
There are no universally required terms (yet)
'''
df_vars = dataset.variables if dataset is not None else get_dataset(filename).variables
df_vars = dict([(k.lower(), v) for k, v in df_vars.items()] )
init_args = {}
dt = {}
return init_args, dt
def _gen_varnames(cls,
filename=None,
dataset=None,
names_dict=None,
std_names_dict=None):
"""
Function to find the default variable names if they are not provided.
:param filename: Name of file that will be searched for variables
:param dataset: Existing instance of a netCDF4.Dataset
:type filename: string
:type dataset: netCDF.Dataset
:return: dict of component to name mapping (eg {'u': 'water_u', 'v': 'water_v', etc})
"""
df = None
if dataset is not None:
df = dataset
else:
df = get_dataset(filename)
if names_dict is None:
names_dict = cls.default_names
if std_names_dict is None:
std_names_dict = cls.cf_names
rd = {}
for k in cls.comp_order:
v = names_dict[k] if k in names_dict else []
for n in v:
if n in df.variables.keys():
rd[k] = n
continue
if k not in rd.keys():
rd[k] = None
for k in cls.comp_order:
v = std_names_dict[k] if k in std_names_dict else []
if rd[k] is None:
for n in v:
for var in df.variables.values():
if (hasattr(var, 'standard_name')
and var.standard_name == n
or hasattr(var, 'long_name')
and var.long_name == n):
rd[k] = var.name
break
return collections.namedtuple('varnames', cls.comp_order)(**rd)
def from_netCDF(cls,
filename=None,
dataset=None,
varname=None,
datavar=None,
tz_offset=None,
**kwargs):
"""
construct a Time object from a netcdf file
:param filename=None: name of netcddf file
:param dataset=None: netcdf dataset object (one or the other)
:param varname=None: name of the netcdf variable
:param datavar=None: Either the time variable name, or
A netcdf variable that needs a Time object.
It will try to find the time variable that
corresponds to the passed in variable.
:param tz_offset=None: offset to adjust for timezone, in hours.
"""
if dataset is None:
dataset = get_dataset(filename)
if datavar is not None:
if hasattr(datavar,
'time') and datavar.time in dataset.dimensions.keys():
varname = datavar.time
else:
varname = datavar.dimensions[
0] if 'time' in datavar.dimensions[0] else None
if varname is None:
return cls.constant_time()
time = cls(data=dataset[varname],
filename=filename,
varname=varname,
tz_offset=tz_offset,
**kwargs)
return time
def from_netCDF(cls,
filename=None,
dataset=None,
varname=None,
datavar=None,
tz_offset=None,
**kwargs):
if dataset is None:
dataset = get_dataset(filename)
if datavar is not None:
if hasattr(datavar,
'time') and datavar.time in dataset.dimensions.keys():
varname = datavar.time
else:
varname = datavar.dimensions[
0] if 'time' in datavar.dimensions[0] else None
if varname is None:
return cls.constant_time()
time = cls(data=dataset[varname],
filename=filename,
varname=varname,
tz_offset=tz_offset,
**kwargs)
return time
def from_netCDF(cls,
filename=None,
varnames=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
depth=None,
data_file=None,
grid_file=None,
dataset=None,
load_all=False,
variables=None,
**kwargs):
'''
Allows one-function creation of a VectorVariable from a file.
:param filename: Default data source. Parameters below take precedence
:param varnames: Names of the variables in the data source file
:param grid_topology: Description of the relationship between grid attributes and variable names.
:param name: Name of property
:param units: Units
:param time: Time axis of the data
:param data: Underlying data source
:param grid: Grid that the data corresponds with
:param dataset: Instance of open Dataset
:param data_file: Name of data source file
:param grid_file: Name of grid source file
:type filename: string
:type varnames: [] of string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Variable = cls._default_component_types['variable']
Depth = cls._default_component_types['depth']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varnames is None:
varnames = cls._gen_varnames(data_file, dataset=ds)
if all([v is None for v in varnames]):
raise ValueError('No compatible variable names found!')
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
data = ds[varnames[0]]
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if time_origin is not None:
time = Time(data=time.data,
filename=data_file,
varname=time.varname,
origin=time_origin)
if depth is None:
if (isinstance(grid, (Grid_S, Grid_R)) and len(data.shape) == 4
or isinstance(grid, Grid_U) and len(data.shape) == 3):
depth = Depth.from_netCDF(
grid_file,
dataset=dg,
)
# if depth is None:
# if (isinstance(grid, Grid_S) and len(data.shape) == 4 or
# (len(data.shape) == 3 and time is None) or
# (isinstance(grid, Grid_U) and len(data.shape) == 3 or
# (len(data.shape) == 2 and time is None))):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if variables is None:
variables = []
for vn in varnames:
if vn is not None:
# Fixme: We're calling from_netCDF from itself ?!?!?
variables.append(
Variable.from_netCDF(filename=filename,
varname=vn,
grid_topology=grid_topology,
units=units,
time=time,
grid=grid,
depth=depth,
data_file=data_file,
grid_file=grid_file,
dataset=ds,
load_all=load_all,
location=None,
**kwargs))
if units is None:
units = [v.units for v in variables]
if all(u == units[0] for u in units):
units = units[0]
return cls(name=name,
filename=filename,
varnames=varnames,
grid_topology=grid_topology,
units=units,
time=time,
grid=grid,
depth=depth,
variables=variables,
data_file=data_file,
grid_file=grid_file,
dataset=ds,
load_all=load_all,
location=None,
**kwargs)
def from_netCDF(cls,
filename=None,
varnames=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
dataset=None,
data_file=None,
grid_file=None,
load_all=False,
bathymetry=None,
zeta=None,
terms=None,
fill_value=0,
**kwargs):
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Variable = cls._default_component_types['variable']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if bathymetry is None:
bathy_name = cls._gen_varname(
filename=filename,
dataset=ds,
names_list=['h'],
std_names_list=['bathymetry at RHO-points'])
bathymetry = Variable.from_netCDF(dataset=ds,
grid=grid,
varname=bathy_name,
name='Bathymetry')
if zeta is None:
zeta_name = cls._gen_varname(filename=filename,
dataset=ds,
names_list=['zeta'],
std_names_list=['free-surface'])
zeta = Variable.from_netCDF(dataset=ds,
grid=grid,
varname=zeta_name,
name='zeta')
if time is None:
time = zeta.time
if terms is None:
terms = {}
for tn, tln in cls.default_terms:
vname = tn
if tn not in ds.variables.keys():
vname = cls._gen_varname(filename, ds, [tn], [tln])
if tn not in ['h',
'zeta']: #don't want to reinclude bathymetry
terms[vname] = ds[vname][:]
return cls(name=name,
time=time,
grid=grid,
bathymetry=bathymetry,
zeta=zeta,
terms=terms,
**kwargs)
def from_netCDF(
cls,
filename=None,
varname=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
depth=None,
dataset=None,
data_file=None,
grid_file=None,
location=None,
load_all=False, # Do we need this? I think not --- maybe a method to fully load later if wanted.
fill_value=0,
**kwargs):
'''
Allows one-function creation of a Variable from a file.
:param filename: Default data source. Has lowest priority;
if dataset, grid_file, or data_file are provided, this function
uses them first
:param varname: Explicit name of the data in the data source file.
Equivalent to the key used to look the item up directly eg 'ds["lon_u"]'
for a netCDF4 Dataset.
:param grid_topology: Description of the relationship between grid attributes
and variable names.
:param name: Name of this object
:param units: string such as 'm/s'
:param time: Time axis of the data. May be a constructed gridded.Time
object, or collection of datetime.datetime objects
:param data: Underlying data object. May be any array-like,
including netCDF4 Variable, etc
:param grid: Grid that the data corresponds with
:param location: The feature where the data aligns with the grid.
:param depth: Depth axis object from gridded.depth
:param dataset: Instance of open netCDF4.Dataset
:param data_file: Name of data source file, if data and grid files are separate
:param grid_file: Name of grid source file, if data and grid files are separate
:type filename: string
:type varname: string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type location: string
:type depth: Depth, S_Depth or L_Depth
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Depth = cls._default_component_types['depth']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if data_file is None:
data_file = os.path.split(ds.filepath())[-1]
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varname is None:
varname = cls._gen_varname(data_file, dataset=ds)
if varname is None:
raise NameError(
'Default current names are not in the data file, '
'must supply variable name')
data = ds[varname]
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if units is None:
try:
units = data.units
except AttributeError:
units = None
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if time_origin is not None:
time = Time(data=time.data,
filename=time.filename,
varname=time.varname,
origin=time_origin)
if depth is None:
if (isinstance(grid, (Grid_S, Grid_R)) and len(data.shape) == 4
or isinstance(grid, Grid_U) and len(data.shape) == 3):
depth = Depth.from_netCDF(
grid_file,
dataset=dg,
)
if location is None:
if hasattr(data, 'location'):
location = data.location
# if len(data.shape) == 4 or (len(data.shape) == 3 and time is None):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if load_all:
data = data[:]
return cls(name=name,
units=units,
time=time,
data=data,
grid=grid,
depth=depth,
grid_file=grid_file,
data_file=data_file,
fill_value=fill_value,
location=location,
varname=varname,
**kwargs)
def env_from_netCDF(filename=None,
dataset=None,
grid_file=None,
data_file=None,
_cls_list=None,
**kwargs):
'''
Returns a list of instances of environment objects that can be produced
from a file or dataset. These instances will be created with a common
underlying grid, and will interconnect when possible.
For example, if an IceAwareWind can find an existing IceConcentration,
it will use it instead of instantiating another. This function tries
ALL gridded types by default. This means if a particular subclass
of object is possible to be built, it is likely that all it's parents
will be built and included as well.
If you wish to limit the types of environment objects that will
be used, pass a list of the types using "_cls_list" kwarg
'''
def attempt_from_netCDF(cls, **klskwargs):
obj = None
try:
obj = c.from_netCDF(**klskwargs)
except Exception as e:
import logging
logging.warn('''Class {0} could not be constituted from netCDF file
Exception: {1}'''.format(c.__name__, e))
return obj
from gnome.environment.gridded_objects_base import Variable, VectorVariable
from gridded.utilities import get_dataset
from gnome.environment import PyGrid, Environment
new_env = []
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
dataset = ds
grid = kwargs.pop('grid', None)
if grid is None:
grid = PyGrid.from_netCDF(filename=filename, dataset=dg, **kwargs)
kwargs['grid'] = grid
if _cls_list is None:
scs = copy.copy(Environment._subclasses)
else:
scs = _cls_list
for c in scs:
if (issubclass(c, (Variable, VectorVariable))
and not any([isinstance(o, c) for o in new_env])):
clskwargs = copy.copy(kwargs)
obj = None
try:
req_refs = c._req_refs
except AttributeError:
req_refs = None
if req_refs is not None:
for ref, klass in req_refs.items():
for o in new_env:
if isinstance(o, klass):
clskwargs[ref] = o
if ref in clskwargs.keys():
continue
else:
obj = attempt_from_netCDF(c,
filename=filename,
dataset=dataset,
grid_file=grid_file,
data_file=data_file,
**clskwargs)
clskwargs[ref] = obj
if obj is not None:
new_env.append(obj)
obj = attempt_from_netCDF(c,
filename=filename,
dataset=dataset,
grid_file=grid_file,
data_file=data_file,
**clskwargs)
if obj is not None:
new_env.append(obj)
return new_env
def env_from_netCDF(filename=None, dataset=None,
grid_file=None, data_file=None, _cls_list=None,
**kwargs):
'''
Returns a list of instances of environment objects that can be produced
from a file or dataset. These instances will be created with a common
underlying grid, and will interconnect when possible.
For example, if an IceAwareWind can find an existing IceConcentration,
it will use it instead of instantiating another. This function tries
ALL gridded types by default. This means if a particular subclass
of object is possible to be built, it is likely that all it's parents
will be built and included as well.
If you wish to limit the types of environment objects that will
be used, pass a list of the types using "_cls_list" kwarg
'''
def attempt_from_netCDF(cls, **klskwargs):
obj = None
try:
obj = c.from_netCDF(**klskwargs)
except Exception as e:
import logging
logging.warn('''Class {0} could not be constituted from netCDF file
Exception: {1}'''.format(c.__name__, e))
return obj
from gnome.environment.gridded_objects_base import Variable, VectorVariable
from gridded.utilities import get_dataset
from gnome.environment import PyGrid, Environment
new_env = []
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
dataset = ds
grid = kwargs.pop('grid', None)
if grid is None:
grid = PyGrid.from_netCDF(filename=filename, dataset=dg, **kwargs)
kwargs['grid'] = grid
if _cls_list is None:
scs = copy.copy(Environment._subclasses)
else:
scs = _cls_list
for c in scs:
if (issubclass(c, (Variable, VectorVariable)) and
not any([isinstance(o, c) for o in new_env])):
clskwargs = copy.copy(kwargs)
obj = None
try:
req_refs = c._req_refs
except AttributeError:
req_refs = None
if req_refs is not None:
for ref, klass in req_refs.items():
for o in new_env:
if isinstance(o, klass):
clskwargs[ref] = o
if ref in clskwargs.keys():
continue
else:
obj = attempt_from_netCDF(c,
filename=filename,
dataset=dataset,
grid_file=grid_file,
data_file=data_file,
**clskwargs)
clskwargs[ref] = obj
if obj is not None:
new_env.append(obj)
obj = attempt_from_netCDF(c,
filename=filename,
dataset=dataset,
grid_file=grid_file,
data_file=data_file,
**clskwargs)
if obj is not None:
new_env.append(obj)
return new_env
def from_netCDF(cls,
filename=None,
varname=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
depth=None,
dataset=None,
data_file=None,
grid_file=None,
load_all=False,
fill_value=0,
**kwargs
):
'''
Allows one-function creation of a Variable from a file.
:param filename: Default data source. Parameters below take precedence
:param varname: Name of the variable in the data source file
:param grid_topology: Description of the relationship between grid attributes and variable names.
:param name: Name of property
:param units: Units
:param time: Time axis of the data
:param data: Underlying data source
:param grid: Grid that the data corresponds with
:param depth: Depth axis object
:param dataset: Instance of open Dataset
:param data_file: Name of data source file
:param grid_file: Name of grid source file
:type filename: string
:type varname: string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type depth: Depth, S_Depth or L_Depth
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Depth = cls._default_component_types['depth']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varname is None:
varname = cls._gen_varname(data_file,
dataset=ds)
if varname is None:
raise NameError('Default current names are not in the data file, must supply variable name')
data = ds[varname]
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if units is None:
try:
units = data.units
except AttributeError:
units = None
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if time_origin is not None:
time = Time(data=time.data, filename=time.filename, varname=time.varname, origin=time_origin)
if depth is None:
if (isinstance(grid, (Grid_S, Grid_R)) and len(data.shape) == 4 or
isinstance(grid, Grid_U) and len(data.shape) == 3):
depth = Depth.from_netCDF(grid_file,
dataset=dg,
)
# if len(data.shape) == 4 or (len(data.shape) == 3 and time is None):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if load_all:
data = data[:]
return cls(name=name,
units=units,
time=time,
data=data,
grid=grid,
depth=depth,
grid_file=grid_file,
data_file=data_file,
fill_value=fill_value,
varname=varname,
**kwargs)
def load_from_varnames(filename,
names_mapping,
attribute_check=None,
post_process=None):
"""
Load a UGrid from a netcdf file where the roles are defined by the
names of the variables.
:param filename: names of the file to load (or OPeNDAP URL).
:param names_mapping: dict that maps the variable names to UGrid components
:param attribute_check=None: list of global attributes that are expected
:type attribute_check: list of tuples to check. Example:
[('grid_type','triangular'),] will check if the
grid_type attribute is set to "triangular"
:param post_process: function to call to do some custom post processing.
it should be a callable that takes (Dataset, UGrid)
The names_mapping dict has to contain at least: 'nodes_lon', 'nodes_lat'
Optionally (and mostly required), it can contain: face_face_connectivity',
'face_coordinates_lon', 'face_coordinates_lat', and 'faces'
"""
ug = UGrid()
attribute_check = {} if attribute_check is None else attribute_check
nc = get_dataset(filename)
# nc = netCDF4.Dataset(filename)
# Check for the specified attributes.
for name, value in attribute_check:
if nc.getncattr(name).lower() != value:
raise ValueError('This does not appear to be a valid file:\n'
'It does not have the "{}"="{}"'
'global attribute set'.format(name, value))
# Nodes.
lon = nc.variables[names_mapping['nodes_lon']]
lat = nc.variables[names_mapping['nodes_lat']]
num_nodes = lon.size
ug.nodes = np.zeros((num_nodes, 2), dtype=lon.dtype)
ug.nodes[:, 0] = lon[:]
ug.nodes[:, 1] = lat[:]
# Faces.
faces = nc.variables[names_mapping['faces']]
# FIXME: This logic assumes there are more than three triangles.
if faces.shape[0] <= faces.shape[1]:
# Fortran order.
faces = faces[:].T
else:
faces = faces[:]
# One-indexed?
if faces.min() == 1:
one_indexed = True
else:
one_indexed = False
if one_indexed:
faces -= 1
ug.faces = faces
# Connectivity (optional).
if 'face_face_connectivity' in names_mapping:
face_face_connectivity = nc.variables[
names_mapping['face_face_connectivity']] # noqa
# FIXME: This logic assumes there are more than three triangles.
if face_face_connectivity.shape[0] <= face_face_connectivity.shape[1]:
# Fortran order.
face_face_connectivity = face_face_connectivity[:].T
else:
face_face_connectivity = face_face_connectivity[:]
if one_indexed:
face_face_connectivity -= 1
ug.face_face_connectivity = face_face_connectivity
# Center (optional).
if ('face_coordinates_lon' in names_mapping
and 'face_coordinates_lat' in names_mapping):
ug.face_coordinates = np.zeros((len(ug.faces), 2), dtype=lon.dtype)
ug.face_coordinates[:, 0] = nc.variables[
names_mapping['face_coordinates_lon']][:] # noqa
ug.face_coordinates[:, 1] = nc.variables[
names_mapping['face_coordinates_lat']][:] # noqa
# Boundaries (optional).
if 'boundaries' in names_mapping:
# FIXME: this one is weird and non-conforming!
# Ignoring the second two fields. What are they?
boundaries = nc.variables[names_mapping['boundaries']][:, :2]
if one_indexed:
boundaries -= 1
ug.boundaries = boundaries
if post_process is not None:
post_process(nc, ug)
return ug
