def test_construction(self, sg_data, sg_topology):
filename = sg_data[0]
dataset = sg_data[1]
grid_topology = sg_topology
sg = Grid_S.from_netCDF(filename, dataset, grid_topology=grid_topology)
assert sg.filename == filename
sg2 = Grid_S.from_netCDF(filename)
assert sg2.filename == filename
sg3 = Grid.from_netCDF(filename, dataset, grid_topology=grid_topology)
sg4 = Grid.from_netCDF(filename)
print(sg3.shape)
print(sg4.shape)
assert sg == sg3
assert sg2 == sg4
def wrapper(*args, **kws):
def _mod(n):
k = kws
s = shared
return (n in s) and ((n not in k) or (n in k and k[n] is None))
if 'filename' in kws and kws['filename'] is not None:
kws['data_file'] = kws['grid_file'] = kws['filename']
ds = dg = None
if _mod('dataset'):
if 'grid_file' in kws and 'data_file' in kws:
if kws['grid_file'] == kws['data_file']:
ds = dg = get_dataset(kws['grid_file'])
else:
ds = get_dataset(kws['data_file'])
dg = get_dataset(kws['grid_file'])
kws['dataset'] = ds
else:
if 'grid_file' in kws and kws['grid_file'] is not None:
dg = get_dataset(kws['grid_file'])
else:
dg = kws['dataset']
ds = kws['dataset']
if _mod('grid'):
gt = kws.get('grid_topology', None)
kws['grid'] = Grid.from_netCDF(kws['filename'], dataset=dg, grid_topology=gt)
if kws.get('varnames', None) is None:
varnames = cls._gen_varnames(kws['data_file'],
dataset=ds)
# if _mod('time'):
# time = Time.from_netCDF(filename=kws['data_file'],
# dataset=ds,
# varname=data)
# kws['time'] = time
return func(*args, **kws)
def __init__(
self,
ncfile=None,
grid=None,
variables=None,
grid_topology=None,
):
"""
Construct a gridded.Dataset object. Can be constructed from a data file,
or also raw grid and variable objects.
:param ncfile: a file to load the Dataset from.
:type ncfile: filename of netcdf file or opendap url or open netCDF4 Dataset object
(could be other file types in the future)
:param grid: a dataset.Grid object or anything that presents the same API.
:param variables: a dict of dataset.Variable objects -- or anything that
presents the same API.
Either a filename or grid and variable objects should be provided -- not both.
"""
if ncfile is not None:
self.nc_dataset = get_dataset(ncfile)
self.filename = self.nc_dataset.filepath()
# self.grid = pyugrid.UGrid.from_nc_dataset(ds)
self.grid = Grid.from_netCDF(filename=self.filename,
dataset=self.nc_dataset)
# var_names = pyugrid.read_netcdf.find_variables(self.nc_dataset,
# self.grid.mesh_name)
self.variables = self._load_variables(self.nc_dataset)
else: # no file passed in -- create from grid and variables
self.filename = None
self.grid = grid
self.variables = variables
def __init__(self,
ncfile=None,
grid=None,
variables=None,
grid_topology=None,
attributes=None):
"""
Construct a gridded.Dataset object. Can be constructed from a data file,
or also raw grid and variable objects.
:param ncfile: A file or files to load the Dataset from.
:type ncfile: Can be one of:
- file path of netcdf file as a string
- opendap url
- list of file paths (uses a netCDF4 MFDataset)
- open netCDF4 Dataset object
(could be other file types in the future)
:param grid: a dataset.Grid object or anything that presents the same API.
:param variables: a dict
of dataset.Variable objects -- or anything that
presents the same API.
:param grid_topology: mapping of grid topology components to netcdf variable names.
used to load non-confirming files.
:type grid_topology: mapping with keys of topology components and values are
variable names.
:param attributes: The global attributes of the dataset -- usually the global
attributes of a netcdf file.
:type attributes: Mapping of attribute name to attributes themselves
(usually strings)
Either a filename or grid and variable objects should be provided -- not both.
If a filename is passed in, the attributes will be pulled from the file, and
the input ones ignored.
"""
if ncfile is not None:
if (grid is not None or variables is not None
or attributes is not None):
raise ValueError(
"You can create a Dataset from a file, or from raw data"
"but not both.")
self.nc_dataset = get_dataset(ncfile)
self.filename = self.nc_dataset.filepath()
self.grid = Grid.from_netCDF(filename=self.filename,
dataset=self.nc_dataset,
grid_topology=grid_topology)
self.variables = self._load_variables(self.nc_dataset)
self.attributes = get_dataset_attrs(self.nc_dataset)
else: # no file passed in -- create from grid and variables
self.filename = None
self.grid = grid
self.variables = variables
self.attributes = {} if attributes is None else attributes
def test_construction(self, ug_data, ug_topology):
filename = ug_data[0]
dataset = ug_data[1]
grid_topology = ug_topology
ug = Grid_U.from_netCDF(filename, dataset, grid_topology=grid_topology)
# assert ug.filename == filename
# assert isinstance(ug.node_lon, nc.Variable)
# assert ug.node_lon.name == 'lonc'
ug2 = Grid_U.from_netCDF(filename)
assert ug2.filename == filename
# assert isinstance(ug2.node_lon, nc.Variable)
# assert ug2.node_lon.name == 'lon'
ug3 = Grid.from_netCDF(filename, dataset, grid_topology=grid_topology)
ug4 = Grid.from_netCDF(filename)
print(ug3.shape)
print(ug4.shape)
assert ug == ug3
assert ug2 == ug4
def test_construction(self, sg_data):
fn = sg_data[0]
sinusoid = sg_data[1]
data = sinusoid['u'][:]
grid = Grid.from_netCDF(dataset=sinusoid)
time = None
u = Variable(name='u',
units='m/s',
data=data,
grid=grid,
time=time,
data_file='staggered_sine_channel.nc',
grid_file='staggered_sine_channel.nc')
curr_file = os.path.join(test_dir, 'staggered_sine_channel.nc')
k = Variable.from_netCDF(filename=curr_file, varname='u', name='u')
assert k.name == u.name
assert k.units == 'm/s'
# fixme: this was failing
# assert k.time == u.time
assert k.data[0, 0] == u.data[0, 0]
def from_netCDF(cls,
filename=None,
varnames=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
depth=None,
data_file=None,
grid_file=None,
dataset=None,
load_all=False,
variables=None,
**kwargs):
'''
Allows one-function creation of a VectorVariable from a file.
:param filename: Default data source. Parameters below take precedence
:param varnames: Names of the variables in the data source file
:param grid_topology: Description of the relationship between grid attributes and variable names.
:param name: Name of property
:param units: Units
:param time: Time axis of the data
:param data: Underlying data source
:param grid: Grid that the data corresponds with
:param dataset: Instance of open Dataset
:param data_file: Name of data source file
:param grid_file: Name of grid source file
:type filename: string
:type varnames: [] of string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Variable = cls._default_component_types['variable']
Depth = cls._default_component_types['depth']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varnames is None:
varnames = cls._gen_varnames(data_file, dataset=ds)
if all([v is None for v in varnames]):
raise ValueError('No compatible variable names found!')
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
data = ds[varnames[0]]
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if time_origin is not None:
time = Time(data=time.data,
filename=data_file,
varname=time.varname,
origin=time_origin)
if depth is None:
if (isinstance(grid, (Grid_S, Grid_R)) and len(data.shape) == 4
or isinstance(grid, Grid_U) and len(data.shape) == 3):
depth = Depth.from_netCDF(
grid_file,
dataset=dg,
)
# if depth is None:
# if (isinstance(grid, Grid_S) and len(data.shape) == 4 or
# (len(data.shape) == 3 and time is None) or
# (isinstance(grid, Grid_U) and len(data.shape) == 3 or
# (len(data.shape) == 2 and time is None))):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if variables is None:
variables = []
for vn in varnames:
if vn is not None:
# Fixme: We're calling from_netCDF from itself ?!?!?
variables.append(
Variable.from_netCDF(filename=filename,
varname=vn,
grid_topology=grid_topology,
units=units,
time=time,
grid=grid,
depth=depth,
data_file=data_file,
grid_file=grid_file,
dataset=ds,
load_all=load_all,
location=None,
**kwargs))
if units is None:
units = [v.units for v in variables]
if all(u == units[0] for u in units):
units = units[0]
return cls(name=name,
filename=filename,
varnames=varnames,
grid_topology=grid_topology,
units=units,
time=time,
grid=grid,
depth=depth,
variables=variables,
data_file=data_file,
grid_file=grid_file,
dataset=ds,
load_all=load_all,
location=None,
**kwargs)
def from_netCDF(
cls,
filename=None,
varname=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
depth=None,
dataset=None,
data_file=None,
grid_file=None,
location=None,
load_all=False, # Do we need this? I think not --- maybe a method to fully load later if wanted.
fill_value=0,
**kwargs):
'''
Allows one-function creation of a Variable from a file.
:param filename: Default data source. Has lowest priority;
if dataset, grid_file, or data_file are provided, this function
uses them first
:param varname: Explicit name of the data in the data source file.
Equivalent to the key used to look the item up directly eg 'ds["lon_u"]'
for a netCDF4 Dataset.
:param grid_topology: Description of the relationship between grid attributes
and variable names.
:param name: Name of this object
:param units: string such as 'm/s'
:param time: Time axis of the data. May be a constructed gridded.Time
object, or collection of datetime.datetime objects
:param data: Underlying data object. May be any array-like,
including netCDF4 Variable, etc
:param grid: Grid that the data corresponds with
:param location: The feature where the data aligns with the grid.
:param depth: Depth axis object from gridded.depth
:param dataset: Instance of open netCDF4.Dataset
:param data_file: Name of data source file, if data and grid files are separate
:param grid_file: Name of grid source file, if data and grid files are separate
:type filename: string
:type varname: string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type location: string
:type depth: Depth, S_Depth or L_Depth
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Depth = cls._default_component_types['depth']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if data_file is None:
data_file = os.path.split(ds.filepath())[-1]
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varname is None:
varname = cls._gen_varname(data_file, dataset=ds)
if varname is None:
raise NameError(
'Default current names are not in the data file, '
'must supply variable name')
data = ds[varname]
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if units is None:
try:
units = data.units
except AttributeError:
units = None
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if time_origin is not None:
time = Time(data=time.data,
filename=time.filename,
varname=time.varname,
origin=time_origin)
if depth is None:
if (isinstance(grid, (Grid_S, Grid_R)) and len(data.shape) == 4
or isinstance(grid, Grid_U) and len(data.shape) == 3):
depth = Depth.from_netCDF(
grid_file,
dataset=dg,
)
if location is None:
if hasattr(data, 'location'):
location = data.location
# if len(data.shape) == 4 or (len(data.shape) == 3 and time is None):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if load_all:
data = data[:]
return cls(name=name,
units=units,
time=time,
data=data,
grid=grid,
depth=depth,
grid_file=grid_file,
data_file=data_file,
fill_value=fill_value,
location=location,
varname=varname,
**kwargs)
def ug():
return Grid.from_netCDF(ug_data()[0], ug_data()[1], grid_topology=ug_topology())
def sg():
return Grid.from_netCDF(sg_data()[0], sg_data()[1], grid_topology=sg_topology())
def from_netCDF(cls,
filename=None,
varname=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
depth=None,
dataset=None,
data_file=None,
grid_file=None,
load_all=False,
fill_value=0,
**kwargs
):
'''
Allows one-function creation of a Variable from a file.
:param filename: Default data source. Parameters below take precedence
:param varname: Name of the variable in the data source file
:param grid_topology: Description of the relationship between grid attributes and variable names.
:param name: Name of property
:param units: Units
:param time: Time axis of the data
:param data: Underlying data source
:param grid: Grid that the data corresponds with
:param depth: Depth axis object
:param dataset: Instance of open Dataset
:param data_file: Name of data source file
:param grid_file: Name of grid source file
:type filename: string
:type varname: string
:type grid_topology: {string : string, ...}
:type name: string
:type units: string
:type time: [] of datetime.datetime, netCDF4 Variable, or Time object
:type data: netCDF4.Variable or numpy.array
:type grid: pysgrid or pyugrid
:type depth: Depth, S_Depth or L_Depth
:type dataset: netCDF4.Dataset
:type data_file: string
:type grid_file: string
'''
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Depth = cls._default_component_types['depth']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if varname is None:
varname = cls._gen_varname(data_file,
dataset=ds)
if varname is None:
raise NameError('Default current names are not in the data file, must supply variable name')
data = ds[varname]
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if units is None:
try:
units = data.units
except AttributeError:
units = None
if time is None:
time = Time.from_netCDF(filename=data_file,
dataset=ds,
datavar=data)
if time_origin is not None:
time = Time(data=time.data, filename=time.filename, varname=time.varname, origin=time_origin)
if depth is None:
if (isinstance(grid, (Grid_S, Grid_R)) and len(data.shape) == 4 or
isinstance(grid, Grid_U) and len(data.shape) == 3):
depth = Depth.from_netCDF(grid_file,
dataset=dg,
)
# if len(data.shape) == 4 or (len(data.shape) == 3 and time is None):
# from gnome.environment.environment_objects import S_Depth
# depth = S_Depth.from_netCDF(grid=grid,
# depth=1,
# data_file=data_file,
# grid_file=grid_file,
# **kwargs)
if load_all:
data = data[:]
return cls(name=name,
units=units,
time=time,
data=data,
grid=grid,
depth=depth,
grid_file=grid_file,
data_file=data_file,
fill_value=fill_value,
varname=varname,
**kwargs)
def from_netCDF(cls,
filename=None,
varnames=None,
grid_topology=None,
name=None,
units=None,
time=None,
time_origin=None,
grid=None,
dataset=None,
data_file=None,
grid_file=None,
load_all=False,
bathymetry=None,
zeta=None,
terms=None,
fill_value=0,
**kwargs):
Grid = cls._default_component_types['grid']
Time = cls._default_component_types['time']
Variable = cls._default_component_types['variable']
if filename is not None:
data_file = filename
grid_file = filename
ds = None
dg = None
if dataset is None:
if grid_file == data_file:
ds = dg = get_dataset(grid_file)
else:
ds = get_dataset(data_file)
dg = get_dataset(grid_file)
else:
if grid_file is not None:
dg = get_dataset(grid_file)
else:
dg = dataset
ds = dataset
if grid is None:
grid = Grid.from_netCDF(grid_file,
dataset=dg,
grid_topology=grid_topology)
if name is None:
name = cls.__name__ + str(cls._def_count)
cls._def_count += 1
if bathymetry is None:
bathy_name = cls._gen_varname(
filename=filename,
dataset=ds,
names_list=['h'],
std_names_list=['bathymetry at RHO-points'])
bathymetry = Variable.from_netCDF(dataset=ds,
grid=grid,
varname=bathy_name,
name='Bathymetry')
if zeta is None:
zeta_name = cls._gen_varname(filename=filename,
dataset=ds,
names_list=['zeta'],
std_names_list=['free-surface'])
zeta = Variable.from_netCDF(dataset=ds,
grid=grid,
varname=zeta_name,
name='zeta')
if time is None:
time = zeta.time
if terms is None:
terms = {}
for tn, tln in cls.default_terms:
vname = tn
if tn not in ds.variables.keys():
vname = cls._gen_varname(filename, ds, [tn], [tln])
if tn not in ['h',
'zeta']: #don't want to reinclude bathymetry
terms[vname] = ds[vname][:]
return cls(name=name,
time=time,
grid=grid,
bathymetry=bathymetry,
zeta=zeta,
terms=terms,
**kwargs)