def from_netCDF(cls,
filename=None,
grid_topology=None,
name=None,
units=None,
time=None,
ice_var=None,
ice_conc_var=None,
grid=None,
dataset=None,
grid_file=None,
data_file=None,
**kwargs):
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = _get_dataset(grid_file)
else:
ds = _get_dataset(data_file)
dg = _get_dataset(grid_file)
else:
ds = dg = dataset
if grid is None:
grid = _init_grid(grid_file,
grid_topology=grid_topology,
dataset=dg)
if ice_var is None:
ice_var = IceVelocity.from_netCDF(filename,
grid=grid,
dataset=ds,
**kwargs)
if time is None:
time = ice_var.time
if ice_conc_var is None:
ice_conc_var = IceConcentration.from_netCDF(filename,
time=time,
grid=grid,
dataset=ds,
**kwargs)
if name is None:
name = 'IceAwareProp'
if units is None:
units = ice_var.units
return cls(name='foo',
units=units,
time=time,
ice_var=ice_var,
ice_conc_var=ice_conc_var,
grid=grid,
grid_file=grid_file,
data_file=data_file,
**kwargs)
def from_netCDF(cls,
filename=None,
grid_topology=None,
name=None,
units=None,
time=None,
ice_var=None,
wind_var=None,
ice_conc_var=None,
grid=None,
dataset=None,
grid_file=None,
data_file=None,
**kwargs):
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = _get_dataset(grid_file)
else:
ds = _get_dataset(data_file)
dg = _get_dataset(grid_file)
else:
ds = dg = dataset
if grid is None:
grid = _init_grid(grid_file,
grid_topology=grid_topology,
dataset=dg)
if wind_var is None:
wind_var = GridWind.from_netCDF(filename,
time=time,
grid=grid,
dataset=ds,
**kwargs)
return super(IceAwareWind, cls).from_netCDF(grid_topology=grid_topology,
name=name,
units=units,
time=time,
ice_var=ice_var,
wind_var=wind_var,
ice_conc_var=ice_conc_var,
grid=grid,
dataset=ds,
grid_file=grid_file,
data_file=data_file,
**kwargs)
def __init__(self,
name=None,
units=None,
time=None,
variables=None,
grid=None,
grid_file=None,
data_file=None,
dataset=None,
**kwargs):
VelocityGrid.__init__(self,
name=name,
units=units,
time=time,
variables=variables,
grid=grid,
grid_file=grid_file,
data_file=data_file,
dataset=dataset)
self.angle = None
df = None
if dataset is not None:
df = dataset
elif grid_file is not None:
df = _get_dataset(grid_file)
if df is not None and 'angle' in df.variables.keys():
# Unrotated ROMS Grid!
self.angle = GriddedProp(name='angle', units='radians', time=None, grid=self.grid, data=df['angle'])
def _gen_varname(cls,
filename=None,
dataset=None,
names_list=None,
std_names_list=None):
"""
Function to find the default variable names if they are not provided.
:param filename: Name of file that will be searched for variables
:param dataset: Existing instance of a netCDF4.Dataset
:type filename: string
:type dataset: netCDF.Dataset
:return: List of default variable names, or None if none are found
"""
df = None
if dataset is not None:
df = dataset
else:
df = _get_dataset(filename)
if names_list is None:
names_list = cls.default_names
for n in names_list:
if n in df.variables.keys():
return n
for n in std_names_list:
for var in df.variables.values():
if hasattr(var, 'standard_name') or hasattr(var, 'long_name'):
if var.name == n:
return n
raise ValueError("Default names not found.")
def _find_topology_var(filename, dataset=None):
gf = _get_dataset(filename, dataset)
gts = []
for v in gf.variables:
if hasattr(v, 'cf_role') and 'topology' in v.cf_role:
gts.append(v)
# gts = gf.get_variables_by_attributes(cf_role=lambda t: t is not None and 'topology' in t)
if len(gts) != 0:
return gts[0]
else:
return None
def wrapper(*args, **kws):
def _mod(n):
k = kws
s = shared
return (n in s) and ((n not in k) or
(n in k and k[n] is None))
if 'filename' in kws and kws['filename'] is not None:
kws['data_file'] = kws['grid_file'] = kws['filename']
if _mod('dataset'):
if 'grid_file' in kws and 'data_file' in kws:
if kws['grid_file'] == kws['data_file']:
ds = dg = _get_dataset(kws['grid_file'])
else:
ds = _get_dataset(kws['data_file'])
dg = _get_dataset(kws['grid_file'])
kws['dataset'] = ds
else:
if 'grid_file' in kws and kws['grid_file'] is not None:
dg = _get_dataset(kws['grid_file'])
else:
dg = kws['dataset']
ds = kws['dataset']
if _mod('grid'):
gt = kws.get('grid_topology', None)
kws['grid'] = PyGrid.from_netCDF(kws['grid_file'],
dataset=dg,
grid_topology=gt)
# if kws.get('varnames', None) is None:
# varnames = cls._gen_varnames(kws['data_file'],
# dataset=ds)
# if _mod('time'):
# time = Time.from_netCDF(filename=kws['data_file'],
# dataset=ds,
# varname=data)
# kws['time'] = time
return func(*args, **kws)
def _find_required_grid_attrs(
cls,
filename,
dataset=None,
grid_topology=None,
):
'''
This function is the top level 'search for attributes' function. If there are any
common attributes to all potential grid types, they will be sought here.
This function returns a dict, which maps an attribute name to a netCDF4
Variable or numpy array object extracted from the dataset. When called from
PyGrid_U or PyGrid_S, this function should provide all the kwargs needed to
create a valid instance.
'''
gf_vars = dataset.variables if dataset is not None else _get_dataset(
filename).variables
init_args = {}
init_args['filename'] = filename
node_attrs = ['node_lon', 'node_lat']
node_coord_names = [['node_lon', 'node_lat'], ['lon', 'lat'],
['lon_psi', 'lat_psi']]
composite_node_names = ['nodes', 'node']
if grid_topology is None:
for n1, n2 in node_coord_names:
if n1 in gf_vars and n2 in gf_vars:
init_args[node_attrs[0]] = gf_vars[n1][:]
init_args[node_attrs[1]] = gf_vars[n2][:]
break
if node_attrs[0] not in init_args:
for n in composite_node_names:
if n in gf_vars:
v = gf_vars[n][:].reshape(-1, 2)
init_args[node_attrs[0]] = v[:, 0]
init_args[node_attrs[1]] = v[:, 1]
break
if node_attrs[0] not in init_args:
raise ValueError('Unable to find node coordinates.')
else:
for n, v in grid_topology.items():
if n in node_attrs:
init_args[n] = gf_vars[v][:]
if n in composite_node_names:
v = gf_vars[n][:].reshape(-1, 2)
init_args[node_attrs[0]] = v[:, 0]
init_args[node_attrs[1]] = v[:, 1]
return init_args, gf_vars
def _gen_varnames(cls,
filename=None,
dataset=None,
names_dict=None,
std_names_dict=None):
"""
Function to find the default variable names if they are not provided.
:param filename: Name of file that will be searched for variables
:param dataset: Existing instance of a netCDF4.Dataset
:type filename: string
:type dataset: netCDF.Dataset
:return: dict of component to name mapping (eg {'u': 'water_u', 'v': 'water_v', etc})
"""
df = None
if dataset is not None:
df = dataset
else:
df = _get_dataset(filename)
if names_dict is None:
names_dict = cls.default_names
if std_names_dict is None:
std_names_dict = cls.cf_names
rd = {}
for k in cls.comp_order:
v = names_dict[k] if k in names_dict else []
for n in v:
if n in df.variables.keys():
rd[k] = n
continue
if k not in rd.keys():
rd[k] = None
for k in cls.comp_order:
v = std_names_dict[k] if k in std_names_dict else []
if rd[k] is None:
for n in v:
for var in df.variables.values():
if (hasattr(var, 'standard_name')
and var.standard_name == n
or hasattr(var, 'long_name')
and var.long_name == n):
rd[k] = var.name
break
return namedtuple('varnames', cls.comp_order)(**rd)
def __init__(self,
bathymetry,
data_file=None,
dataset=None,
terms={},
**kwargs):
ds = dataset
if ds is None:
if data_file is None:
data_file = bathymetry.data_file
if data_file is None:
raise ValueError("Need data_file or dataset containing sigma equation terms")
ds = _get_dataset(data_file)
self.bathymetry = bathymetry
self.terms = terms
if len(terms) == 0:
for s in S_Depth.default_terms:
for term in s:
self.terms[term] = ds[term][:]
def __init__(self,
bathymetry,
data_file=None,
dataset=None,
terms={},
**kwargs):
ds = dataset
if ds is None:
if data_file is None:
data_file = bathymetry.data_file
if data_file is None:
raise ValueError("Need data_file or dataset containing sigma equation terms")
ds = _get_dataset(data_file)
self.bathymetry = bathymetry
self.terms = terms
if len(terms) == 0:
for s in S_Depth_T1.default_terms:
for term in s:
self.terms[term] = ds[term][:]
def _gen_varnames(cls,
filename=None,
dataset=None):
"""
Function to find the default variable names if they are not provided.
:param filename: Name of file that will be searched for variables
:param dataset: Existing instance of a netCDF4.Dataset
:type filename: string
:type dataset: netCDF.Dataset
:return: List of default variable names, or None if none are found
"""
df = None
if dataset is not None:
df = dataset
else:
df = _get_dataset(filename)
for n in cls.default_names:
if all([sn in df.variables.keys() for sn in n]):
return n
raise ValueError("Default names not found.")
def from_netCDF(cls,
filename=None,
dataset=None,
varname=None,
datavar=None,
tz_offset=None,
**kwargs):
if dataset is None:
dataset = _get_dataset(filename)
if datavar is not None:
if hasattr(datavar, 'time') and datavar.time in dataset.dimensions.keys():
varname = datavar.time
else:
varname = datavar.dimensions[0] if 'time' in datavar.dimensions[0] else None
if varname is None:
return None
time = cls(time=dataset[varname],
filename=filename,
varname=varname,
tz_offset=tz_offset,
**kwargs
)
return time
def from_netCDF(cls,
filename=None,
dataset=None,
grid_type=None,
grid_topology=None,
*args,
**kwargs):
'''
:param filename: File containing a grid
:param dataset: Takes precedence over filename, if provided.
:param grid_type: Must be provided if Dataset does not have a 'grid_type' attribute, or valid topology variable
:param grid_topology: A dictionary mapping of grid attribute to variable name. Takes precendence over discovered attributes
:param **kwargs: All kwargs to SGrid or UGrid are valid, and take precedence over all.
:returns: Instance of PyGrid_U, PyGrid_S, or PyGrid_R
'''
gf = dataset if filename is None else _get_dataset(filename, dataset)
if gf is None:
raise ValueError('No filename or dataset provided')
cls = PyGrid._get_grid_type(gf, grid_topology, grid_type)
init_args, gf_vars = cls._find_required_grid_attrs(
filename, dataset=dataset, grid_topology=grid_topology)
return cls(**init_args)
def __init__(self, angle=None, **kwargs):
"""
:param angle: scalar field of cell rotation angles (for rotated/distorted grids)
"""
if 'variables' in kwargs:
variables = kwargs['variables']
if len(variables) == 2:
variables.append(TimeSeriesProp(name='constant w', data=[0.0], time=Time.constant_time(), units='m/s'))
kwargs['variables'] = variables
if angle is None:
df = None
if kwargs.get('dataset', None) is not None:
df = kwargs['dataset']
elif kwargs.get('grid_file', None) is not None:
df = _get_dataset(kwargs['grid_file'])
if df is not None and 'angle' in df.variables.keys():
# Unrotated ROMS Grid!
self.angle = GriddedProp(name='angle', units='radians', time=None, grid=kwargs['grid'], data=df['angle'])
else:
self.angle = None
else:
self.angle = angle
super(VelocityGrid, self).__init__(**kwargs)
def from_netCDF(cls,
filename=None,
grid_topology=None,
name=None,
temperature=None,
salinity=None,
sediment=None,
grid=None,
dataset=None,
grid_file=None,
data_file=None):
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = _get_dataset(grid_file)
else:
ds = _get_dataset(data_file)
dg = _get_dataset(grid_file)
else:
ds = dg = dataset
if grid is None:
grid = _init_grid(grid_file,
grid_topology=grid_topology,
dataset=dg)
if time is None:
time = ice_var.time
if temperature is None:
try:
temperature = GridTemperature.from_netCDF(filename,
time=time,
grid=grid,
dataset=ds)
except:
temperature = 300.
if salinity is None:
try:
salinity = GridSalinity.from_netCDF(filename,
time=time,
grid=grid,
dataset=ds)
except:
salinity = 35.
if sediment is None:
try:
sediment = GridSediment.from_netCDF(filename,
time=time,
grid=grid,
dataset=ds)
except:
sediment = .005
if name is None:
name = 'WaterConditions'
if units is None:
units = water_var.units
return cls(name=name,
units=units,
time=time,
ice_var=ice_var,
water_var=water_var,
ice_conc_var=ice_conc_var,
grid=grid,
grid_file=grid_file,
data_file=data_file)
def env_from_netCDF(filename=None,
dataset=None,
grid_file=None,
data_file=None,
_cls_list=None,
**kwargs):
'''
Returns a list of instances of environment objects that can be produced from a file or dataset.
These instances will be created with a common underlying grid, and will interconnect when possible
For example, if an IceAwareWind can find an existing IceConcentration, it will use it instead of
instantiating another. This function tries ALL gridded types by default. This means if a particular
subclass of object is possible to be built, it is likely that all it's parents will be built and included
as well.
If you wish to limit the types of environment objects that will be used, pass a list of the types
using "_cls_list" kwarg'''
def attempt_from_netCDF(cls, **klskwargs):
obj = None
try:
obj = c.from_netCDF(**klskwargs)
except Exception as e:
import logging
logging.warn('''Class {0} could not be constituted from netCDF file
Exception: {1}'''.format(c.__name__, e))
return obj
from gnome.utilities.file_tools.data_helpers import _get_dataset
from gnome.environment.environment_objects import GriddedProp, GridVectorProp
from gnome.environment import PyGrid, Environment
import copy
new_env = []
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = _get_dataset(grid_file)
else:
ds = _get_dataset(data_file)
dg = _get_dataset(grid_file)
else:
if grid_file is not None:
dg = _get_dataset(grid_file)
else:
dg = dataset
ds = dataset
dataset = ds
grid = kwargs.pop('grid', None)
if grid is None:
grid = PyGrid.from_netCDF(filename=filename, dataset=dg, **kwargs)
kwargs['grid'] = grid
scs = copy.copy(
Environment._subclasses) if _cls_list is None else _cls_list
for c in scs:
if issubclass(c, (GriddedProp, GridVectorProp)) and not any(
[isinstance(o, c) for o in new_env]):
clskwargs = copy.copy(kwargs)
obj = None
try:
req_refs = c._req_refs
except AttributeError:
req_refs = None
if req_refs is not None:
for ref, klass in req_refs.items():
for o in new_env:
if isinstance(o, klass):
clskwargs[ref] = o
if ref in clskwargs.keys():
continue
else:
obj = attempt_from_netCDF(c,
filename=filename,
dataset=dataset,
grid_file=grid_file,
data_file=data_file,
**clskwargs)
clskwargs[ref] = obj
if obj is not None:
new_env.append(obj)
obj = attempt_from_netCDF(c,
filename=filename,
dataset=dataset,
grid_file=grid_file,
data_file=data_file,
**clskwargs)
if obj is not None:
new_env.append(obj)
return new_env
def from_netCDF(cls,
filename=None,
varname=None,
grid_topology=None,
name=None,
units=None,
time=None,
grid=None,
depth=None,
dataset=None,
data_file=None,
grid_file=None,
load_all=False,
fill_value=0,
**kwargs
):
'''
Allows one-function creation of a GriddedProp from a file.
:param filename: Default data source. Parameters below take precedence
:param varname: Name of the variable in the data source file
:param grid_topology: Description of the relationship between grid attributes and variable names.
:param name: Name of property
:param units: Units
:param time: Time axis of the data
:param data: Underlying data source
:param grid: Grid that the data corresponds with
:param depth: Depth axis object
:param dataset: Instance of open Dataset
:param data_file: Name of data source file
:param grid_file: Name of grid source file
:type filename: string
:type varname: string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type S_Depth or L_Depth
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = _get_dataset(grid_file)
else:
ds = _get_dataset(data_file)
dg = _get_dataset(grid_file)
else:
if grid_file is not None:
dg = _get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = _init_grid(grid_file,
grid_topology=grid_topology,
dataset=dg)
if varname is None:
varname = cls._gen_varname(data_file,
dataset=ds)
if varname is None:
raise NameError('Default current names are not in the data file, must supply variable name')
data = ds[varname]
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if units is None:
try:
units = data.units
except AttributeError:
units = None
timevar = None
if time is None:
try:
timevar = data.time if data.time == data.dimensions[0] else data.dimensions[0]
except AttributeError:
if len(data.dimensions) > 2:
timevar = data.dimensions[0]
time = Time(ds[timevar])
else:
time = None
if depth is None:
from gnome.environment.environment_objects import Depth
depth = Depth(surface_index=-1)
# if len(data.shape) == 4 or (len(data.shape) == 3 and time is None):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if load_all:
data = data[:]
return cls(name=name,
units=units,
time=time,
data=data,
grid=grid,
depth=depth,
grid_file=grid_file,
data_file=data_file,
fill_value=fill_value,
**kwargs)
def from_netCDF(cls,
filename=None,
varname=None,
grid_topology=None,
name=None,
units=None,
time=None,
grid=None,
depth=None,
dataset=None,
data_file=None,
grid_file=None,
load_all=False,
fill_value=0,
**kwargs):
'''
Allows one-function creation of a GriddedProp from a file.
:param filename: Default data source. Parameters below take precedence
:param varname: Name of the variable in the data source file
:param grid_topology: Description of the relationship between grid attributes and variable names.
:param name: Name of property
:param units: Units
:param time: Time axis of the data
:param data: Underlying data source
:param grid: Grid that the data corresponds with
:param depth: Depth axis object
:param dataset: Instance of open Dataset
:param data_file: Name of data source file
:param grid_file: Name of grid source file
:type filename: string
:type varname: string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type depth: Depth, S_Depth or L_Depth
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = _get_dataset(grid_file)
else:
ds = _get_dataset(data_file)
dg = _get_dataset(grid_file)
else:
if grid_file is not None:
dg = _get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = PyGrid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varname is None:
varname = cls._gen_varname(data_file, dataset=ds)
if varname is None:
raise NameError(
'Default current names are not in the data file, must supply variable name'
)
data = ds[varname]
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if units is None:
try:
units = data.units
except AttributeError:
units = None
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if depth is None:
if (isinstance(grid, PyGrid_S) and len(data.shape) == 4
or isinstance(grid, PyGrid_U) and len(data.shape) == 3):
from gnome.environment.environment_objects import Depth
depth = Depth(surface_index=-1)
# if len(data.shape) == 4 or (len(data.shape) == 3 and time is None):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if load_all:
data = data[:]
return cls(name=name,
units=units,
time=time,
data=data,
grid=grid,
depth=depth,
grid_file=grid_file,
data_file=data_file,
fill_value=fill_value,
varname=varname,
**kwargs)
def from_netCDF(cls,
filename=None,
varnames=None,
grid_topology=None,
name=None,
units=None,
time=None,
grid=None,
depth=None,
data_file=None,
grid_file=None,
dataset=None,
load_all=False,
**kwargs):
'''
Allows one-function creation of a GridVectorProp from a file.
:param filename: Default data source. Parameters below take precedence
:param varnames: Names of the variables in the data source file
:param grid_topology: Description of the relationship between grid attributes and variable names.
:param name: Name of property
:param units: Units
:param time: Time axis of the data
:param data: Underlying data source
:param grid: Grid that the data corresponds with
:param dataset: Instance of open Dataset
:param data_file: Name of data source file
:param grid_file: Name of grid source file
:type filename: string
:type varnames: [] of string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = _get_dataset(grid_file)
else:
ds = _get_dataset(data_file)
dg = _get_dataset(grid_file)
else:
if grid_file is not None:
dg = _get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = PyGrid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varnames is None:
varnames = cls._gen_varnames(data_file, dataset=ds)
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
data = ds[varnames[0]]
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if depth is None:
if (isinstance(grid, PyGrid_S) and len(data.shape) == 4
or (len(data.shape) == 3 and time is None)
or (isinstance(grid, PyGrid_U) and len(data.shape) == 3 or
(len(data.shape) == 2 and time is None))):
from gnome.environment.environment_objects import Depth
depth = Depth(surface_index=-1)
# if len(data.shape) == 4 or (len(data.shape) == 3 and time is None):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
variables = OrderedCollection(dtype=EnvProp)
for vn in varnames:
if vn is not None:
variables.append(
GriddedProp.from_netCDF(filename=filename,
varname=vn,
grid_topology=grid_topology,
units=units,
time=time,
grid=grid,
depth=depth,
data_file=data_file,
grid_file=grid_file,
dataset=ds,
load_all=load_all,
**kwargs))
if units is None:
units = [v.units for v in variables]
if all(u == units[0] for u in units):
units = units[0]
return cls(name=name,
filename=filename,
varnames=varnames,
grid_topology=grid_topology,
units=units,
time=time,
grid=grid,
depth=depth,
variables=variables,
data_file=data_file,
grid_file=grid_file,
dataset=ds,
load_all=load_all,
**kwargs)
