def from_netCDF(cls, filename=None, grid_topology=None, name=None, units=None, time=None, ice_var=None, ice_conc_var=None, grid=None, dataset=None, grid_file=None, data_file=None, **kwargs): if filename is not None: data_file = filename grid_file = filename ds = None dg = None if dataset is None: if grid_file == data_file: ds = dg = _get_dataset(grid_file) else: ds = _get_dataset(data_file) dg = _get_dataset(grid_file) else: ds = dg = dataset if grid is None: grid = _init_grid(grid_file, grid_topology=grid_topology, dataset=dg) if ice_var is None: ice_var = IceVelocity.from_netCDF(filename, grid=grid, dataset=ds, **kwargs) if time is None: time = ice_var.time if ice_conc_var is None: ice_conc_var = IceConcentration.from_netCDF(filename, time=time, grid=grid, dataset=ds, **kwargs) if name is None: name = 'IceAwareProp' if units is None: units = ice_var.units return cls(name='foo', units=units, time=time, ice_var=ice_var, ice_conc_var=ice_conc_var, grid=grid, grid_file=grid_file, data_file=data_file, **kwargs)
def from_netCDF(cls, filename=None, grid_topology=None, name=None, units=None, time=None, ice_var=None, wind_var=None, ice_conc_var=None, grid=None, dataset=None, grid_file=None, data_file=None, **kwargs): if filename is not None: data_file = filename grid_file = filename ds = None dg = None if dataset is None: if grid_file == data_file: ds = dg = _get_dataset(grid_file) else: ds = _get_dataset(data_file) dg = _get_dataset(grid_file) else: ds = dg = dataset if grid is None: grid = _init_grid(grid_file, grid_topology=grid_topology, dataset=dg) if wind_var is None: wind_var = GridWind.from_netCDF(filename, time=time, grid=grid, dataset=ds, **kwargs) return super(IceAwareWind, cls).from_netCDF(grid_topology=grid_topology, name=name, units=units, time=time, ice_var=ice_var, wind_var=wind_var, ice_conc_var=ice_conc_var, grid=grid, dataset=ds, grid_file=grid_file, data_file=data_file, **kwargs)
def __init__(self, name=None, units=None, time=None, variables=None, grid=None, grid_file=None, data_file=None, dataset=None, **kwargs): VelocityGrid.__init__(self, name=name, units=units, time=time, variables=variables, grid=grid, grid_file=grid_file, data_file=data_file, dataset=dataset) self.angle = None df = None if dataset is not None: df = dataset elif grid_file is not None: df = _get_dataset(grid_file) if df is not None and 'angle' in df.variables.keys(): # Unrotated ROMS Grid! self.angle = GriddedProp(name='angle', units='radians', time=None, grid=self.grid, data=df['angle'])
def _gen_varname(cls, filename=None, dataset=None, names_list=None, std_names_list=None): """ Function to find the default variable names if they are not provided. :param filename: Name of file that will be searched for variables :param dataset: Existing instance of a netCDF4.Dataset :type filename: string :type dataset: netCDF.Dataset :return: List of default variable names, or None if none are found """ df = None if dataset is not None: df = dataset else: df = _get_dataset(filename) if names_list is None: names_list = cls.default_names for n in names_list: if n in df.variables.keys(): return n for n in std_names_list: for var in df.variables.values(): if hasattr(var, 'standard_name') or hasattr(var, 'long_name'): if var.name == n: return n raise ValueError("Default names not found.")
def _find_topology_var(filename, dataset=None): gf = _get_dataset(filename, dataset) gts = [] for v in gf.variables: if hasattr(v, 'cf_role') and 'topology' in v.cf_role: gts.append(v) # gts = gf.get_variables_by_attributes(cf_role=lambda t: t is not None and 'topology' in t) if len(gts) != 0: return gts[0] else: return None
def wrapper(*args, **kws): def _mod(n): k = kws s = shared return (n in s) and ((n not in k) or (n in k and k[n] is None)) if 'filename' in kws and kws['filename'] is not None: kws['data_file'] = kws['grid_file'] = kws['filename'] if _mod('dataset'): if 'grid_file' in kws and 'data_file' in kws: if kws['grid_file'] == kws['data_file']: ds = dg = _get_dataset(kws['grid_file']) else: ds = _get_dataset(kws['data_file']) dg = _get_dataset(kws['grid_file']) kws['dataset'] = ds else: if 'grid_file' in kws and kws['grid_file'] is not None: dg = _get_dataset(kws['grid_file']) else: dg = kws['dataset'] ds = kws['dataset'] if _mod('grid'): gt = kws.get('grid_topology', None) kws['grid'] = PyGrid.from_netCDF(kws['grid_file'], dataset=dg, grid_topology=gt) # if kws.get('varnames', None) is None: # varnames = cls._gen_varnames(kws['data_file'], # dataset=ds) # if _mod('time'): # time = Time.from_netCDF(filename=kws['data_file'], # dataset=ds, # varname=data) # kws['time'] = time return func(*args, **kws)
def _find_required_grid_attrs( cls, filename, dataset=None, grid_topology=None, ): ''' This function is the top level 'search for attributes' function. If there are any common attributes to all potential grid types, they will be sought here. This function returns a dict, which maps an attribute name to a netCDF4 Variable or numpy array object extracted from the dataset. When called from PyGrid_U or PyGrid_S, this function should provide all the kwargs needed to create a valid instance. ''' gf_vars = dataset.variables if dataset is not None else _get_dataset( filename).variables init_args = {} init_args['filename'] = filename node_attrs = ['node_lon', 'node_lat'] node_coord_names = [['node_lon', 'node_lat'], ['lon', 'lat'], ['lon_psi', 'lat_psi']] composite_node_names = ['nodes', 'node'] if grid_topology is None: for n1, n2 in node_coord_names: if n1 in gf_vars and n2 in gf_vars: init_args[node_attrs[0]] = gf_vars[n1][:] init_args[node_attrs[1]] = gf_vars[n2][:] break if node_attrs[0] not in init_args: for n in composite_node_names: if n in gf_vars: v = gf_vars[n][:].reshape(-1, 2) init_args[node_attrs[0]] = v[:, 0] init_args[node_attrs[1]] = v[:, 1] break if node_attrs[0] not in init_args: raise ValueError('Unable to find node coordinates.') else: for n, v in grid_topology.items(): if n in node_attrs: init_args[n] = gf_vars[v][:] if n in composite_node_names: v = gf_vars[n][:].reshape(-1, 2) init_args[node_attrs[0]] = v[:, 0] init_args[node_attrs[1]] = v[:, 1] return init_args, gf_vars
def _gen_varnames(cls, filename=None, dataset=None, names_dict=None, std_names_dict=None): """ Function to find the default variable names if they are not provided. :param filename: Name of file that will be searched for variables :param dataset: Existing instance of a netCDF4.Dataset :type filename: string :type dataset: netCDF.Dataset :return: dict of component to name mapping (eg {'u': 'water_u', 'v': 'water_v', etc}) """ df = None if dataset is not None: df = dataset else: df = _get_dataset(filename) if names_dict is None: names_dict = cls.default_names if std_names_dict is None: std_names_dict = cls.cf_names rd = {} for k in cls.comp_order: v = names_dict[k] if k in names_dict else [] for n in v: if n in df.variables.keys(): rd[k] = n continue if k not in rd.keys(): rd[k] = None for k in cls.comp_order: v = std_names_dict[k] if k in std_names_dict else [] if rd[k] is None: for n in v: for var in df.variables.values(): if (hasattr(var, 'standard_name') and var.standard_name == n or hasattr(var, 'long_name') and var.long_name == n): rd[k] = var.name break return namedtuple('varnames', cls.comp_order)(**rd)
def __init__(self, bathymetry, data_file=None, dataset=None, terms={}, **kwargs): ds = dataset if ds is None: if data_file is None: data_file = bathymetry.data_file if data_file is None: raise ValueError("Need data_file or dataset containing sigma equation terms") ds = _get_dataset(data_file) self.bathymetry = bathymetry self.terms = terms if len(terms) == 0: for s in S_Depth.default_terms: for term in s: self.terms[term] = ds[term][:]
def __init__(self, bathymetry, data_file=None, dataset=None, terms={}, **kwargs): ds = dataset if ds is None: if data_file is None: data_file = bathymetry.data_file if data_file is None: raise ValueError("Need data_file or dataset containing sigma equation terms") ds = _get_dataset(data_file) self.bathymetry = bathymetry self.terms = terms if len(terms) == 0: for s in S_Depth_T1.default_terms: for term in s: self.terms[term] = ds[term][:]
def _gen_varnames(cls, filename=None, dataset=None): """ Function to find the default variable names if they are not provided. :param filename: Name of file that will be searched for variables :param dataset: Existing instance of a netCDF4.Dataset :type filename: string :type dataset: netCDF.Dataset :return: List of default variable names, or None if none are found """ df = None if dataset is not None: df = dataset else: df = _get_dataset(filename) for n in cls.default_names: if all([sn in df.variables.keys() for sn in n]): return n raise ValueError("Default names not found.")
def from_netCDF(cls, filename=None, dataset=None, varname=None, datavar=None, tz_offset=None, **kwargs): if dataset is None: dataset = _get_dataset(filename) if datavar is not None: if hasattr(datavar, 'time') and datavar.time in dataset.dimensions.keys(): varname = datavar.time else: varname = datavar.dimensions[0] if 'time' in datavar.dimensions[0] else None if varname is None: return None time = cls(time=dataset[varname], filename=filename, varname=varname, tz_offset=tz_offset, **kwargs ) return time
def from_netCDF(cls, filename=None, dataset=None, grid_type=None, grid_topology=None, *args, **kwargs): ''' :param filename: File containing a grid :param dataset: Takes precedence over filename, if provided. :param grid_type: Must be provided if Dataset does not have a 'grid_type' attribute, or valid topology variable :param grid_topology: A dictionary mapping of grid attribute to variable name. Takes precendence over discovered attributes :param **kwargs: All kwargs to SGrid or UGrid are valid, and take precedence over all. :returns: Instance of PyGrid_U, PyGrid_S, or PyGrid_R ''' gf = dataset if filename is None else _get_dataset(filename, dataset) if gf is None: raise ValueError('No filename or dataset provided') cls = PyGrid._get_grid_type(gf, grid_topology, grid_type) init_args, gf_vars = cls._find_required_grid_attrs( filename, dataset=dataset, grid_topology=grid_topology) return cls(**init_args)
def __init__(self, angle=None, **kwargs): """ :param angle: scalar field of cell rotation angles (for rotated/distorted grids) """ if 'variables' in kwargs: variables = kwargs['variables'] if len(variables) == 2: variables.append(TimeSeriesProp(name='constant w', data=[0.0], time=Time.constant_time(), units='m/s')) kwargs['variables'] = variables if angle is None: df = None if kwargs.get('dataset', None) is not None: df = kwargs['dataset'] elif kwargs.get('grid_file', None) is not None: df = _get_dataset(kwargs['grid_file']) if df is not None and 'angle' in df.variables.keys(): # Unrotated ROMS Grid! self.angle = GriddedProp(name='angle', units='radians', time=None, grid=kwargs['grid'], data=df['angle']) else: self.angle = None else: self.angle = angle super(VelocityGrid, self).__init__(**kwargs)
def from_netCDF(cls, filename=None, grid_topology=None, name=None, temperature=None, salinity=None, sediment=None, grid=None, dataset=None, grid_file=None, data_file=None): if filename is not None: data_file = filename grid_file = filename ds = None dg = None if dataset is None: if grid_file == data_file: ds = dg = _get_dataset(grid_file) else: ds = _get_dataset(data_file) dg = _get_dataset(grid_file) else: ds = dg = dataset if grid is None: grid = _init_grid(grid_file, grid_topology=grid_topology, dataset=dg) if time is None: time = ice_var.time if temperature is None: try: temperature = GridTemperature.from_netCDF(filename, time=time, grid=grid, dataset=ds) except: temperature = 300. if salinity is None: try: salinity = GridSalinity.from_netCDF(filename, time=time, grid=grid, dataset=ds) except: salinity = 35. if sediment is None: try: sediment = GridSediment.from_netCDF(filename, time=time, grid=grid, dataset=ds) except: sediment = .005 if name is None: name = 'WaterConditions' if units is None: units = water_var.units return cls(name=name, units=units, time=time, ice_var=ice_var, water_var=water_var, ice_conc_var=ice_conc_var, grid=grid, grid_file=grid_file, data_file=data_file)
def env_from_netCDF(filename=None, dataset=None, grid_file=None, data_file=None, _cls_list=None, **kwargs): ''' Returns a list of instances of environment objects that can be produced from a file or dataset. These instances will be created with a common underlying grid, and will interconnect when possible For example, if an IceAwareWind can find an existing IceConcentration, it will use it instead of instantiating another. This function tries ALL gridded types by default. This means if a particular subclass of object is possible to be built, it is likely that all it's parents will be built and included as well. If you wish to limit the types of environment objects that will be used, pass a list of the types using "_cls_list" kwarg''' def attempt_from_netCDF(cls, **klskwargs): obj = None try: obj = c.from_netCDF(**klskwargs) except Exception as e: import logging logging.warn('''Class {0} could not be constituted from netCDF file Exception: {1}'''.format(c.__name__, e)) return obj from gnome.utilities.file_tools.data_helpers import _get_dataset from gnome.environment.environment_objects import GriddedProp, GridVectorProp from gnome.environment import PyGrid, Environment import copy new_env = [] if filename is not None: data_file = filename grid_file = filename ds = None dg = None if dataset is None: if grid_file == data_file: ds = dg = _get_dataset(grid_file) else: ds = _get_dataset(data_file) dg = _get_dataset(grid_file) else: if grid_file is not None: dg = _get_dataset(grid_file) else: dg = dataset ds = dataset dataset = ds grid = kwargs.pop('grid', None) if grid is None: grid = PyGrid.from_netCDF(filename=filename, dataset=dg, **kwargs) kwargs['grid'] = grid scs = copy.copy( Environment._subclasses) if _cls_list is None else _cls_list for c in scs: if issubclass(c, (GriddedProp, GridVectorProp)) and not any( [isinstance(o, c) for o in new_env]): clskwargs = copy.copy(kwargs) obj = None try: req_refs = c._req_refs except AttributeError: req_refs = None if req_refs is not None: for ref, klass in req_refs.items(): for o in new_env: if isinstance(o, klass): clskwargs[ref] = o if ref in clskwargs.keys(): continue else: obj = attempt_from_netCDF(c, filename=filename, dataset=dataset, grid_file=grid_file, data_file=data_file, **clskwargs) clskwargs[ref] = obj if obj is not None: new_env.append(obj) obj = attempt_from_netCDF(c, filename=filename, dataset=dataset, grid_file=grid_file, data_file=data_file, **clskwargs) if obj is not None: new_env.append(obj) return new_env
def from_netCDF(cls, filename=None, varname=None, grid_topology=None, name=None, units=None, time=None, grid=None, depth=None, dataset=None, data_file=None, grid_file=None, load_all=False, fill_value=0, **kwargs ): ''' Allows one-function creation of a GriddedProp from a file. :param filename: Default data source. Parameters below take precedence :param varname: Name of the variable in the data source file :param grid_topology: Description of the relationship between grid attributes and variable names. :param name: Name of property :param units: Units :param time: Time axis of the data :param data: Underlying data source :param grid: Grid that the data corresponds with :param depth: Depth axis object :param dataset: Instance of open Dataset :param data_file: Name of data source file :param grid_file: Name of grid source file :type filename: string :type varname: string :type grid_topology: {string : string, ...} :type name: string :type units: string :type time: [] of datetime.datetime, netCDF4 Variable, or Time object :type data: netCDF4.Variable or numpy.array :type grid: pysgrid or pyugrid :type S_Depth or L_Depth :type dataset: netCDF4.Dataset :type data_file: string :type grid_file: string ''' if filename is not None: data_file = filename grid_file = filename ds = None dg = None if dataset is None: if grid_file == data_file: ds = dg = _get_dataset(grid_file) else: ds = _get_dataset(data_file) dg = _get_dataset(grid_file) else: if grid_file is not None: dg = _get_dataset(grid_file) else: dg = dataset ds = dataset if grid is None: grid = _init_grid(grid_file, grid_topology=grid_topology, dataset=dg) if varname is None: varname = cls._gen_varname(data_file, dataset=ds) if varname is None: raise NameError('Default current names are not in the data file, must supply variable name') data = ds[varname] if name is None: name = cls.__name__ + str(cls._def_count) cls._def_count += 1 if units is None: try: units = data.units except AttributeError: units = None timevar = None if time is None: try: timevar = data.time if data.time == data.dimensions[0] else data.dimensions[0] except AttributeError: if len(data.dimensions) > 2: timevar = data.dimensions[0] time = Time(ds[timevar]) else: time = None if depth is None: from gnome.environment.environment_objects import Depth depth = Depth(surface_index=-1) # if len(data.shape) == 4 or (len(data.shape) == 3 and time is None): # from gnome.environment.environment_objects import S_Depth # depth = S_Depth.from_netCDF(grid=grid, # depth=1, # data_file=data_file, # grid_file=grid_file, # **kwargs) if load_all: data = data[:] return cls(name=name, units=units, time=time, data=data, grid=grid, depth=depth, grid_file=grid_file, data_file=data_file, fill_value=fill_value, **kwargs)
def from_netCDF(cls, filename=None, varname=None, grid_topology=None, name=None, units=None, time=None, grid=None, depth=None, dataset=None, data_file=None, grid_file=None, load_all=False, fill_value=0, **kwargs): ''' Allows one-function creation of a GriddedProp from a file. :param filename: Default data source. Parameters below take precedence :param varname: Name of the variable in the data source file :param grid_topology: Description of the relationship between grid attributes and variable names. :param name: Name of property :param units: Units :param time: Time axis of the data :param data: Underlying data source :param grid: Grid that the data corresponds with :param depth: Depth axis object :param dataset: Instance of open Dataset :param data_file: Name of data source file :param grid_file: Name of grid source file :type filename: string :type varname: string :type grid_topology: {string : string, ...} :type name: string :type units: string :type time: [] of datetime.datetime, netCDF4 Variable, or Time object :type data: netCDF4.Variable or numpy.array :type grid: pysgrid or pyugrid :type depth: Depth, S_Depth or L_Depth :type dataset: netCDF4.Dataset :type data_file: string :type grid_file: string ''' if filename is not None: data_file = filename grid_file = filename ds = None dg = None if dataset is None: if grid_file == data_file: ds = dg = _get_dataset(grid_file) else: ds = _get_dataset(data_file) dg = _get_dataset(grid_file) else: if grid_file is not None: dg = _get_dataset(grid_file) else: dg = dataset ds = dataset if grid is None: grid = PyGrid.from_netCDF(grid_file, dataset=dg, grid_topology=grid_topology) if varname is None: varname = cls._gen_varname(data_file, dataset=ds) if varname is None: raise NameError( 'Default current names are not in the data file, must supply variable name' ) data = ds[varname] if name is None: name = cls.__name__ + str(cls._def_count) cls._def_count += 1 if units is None: try: units = data.units except AttributeError: units = None if time is None: time = Time.from_netCDF(filename=data_file, dataset=ds, datavar=data) if depth is None: if (isinstance(grid, PyGrid_S) and len(data.shape) == 4 or isinstance(grid, PyGrid_U) and len(data.shape) == 3): from gnome.environment.environment_objects import Depth depth = Depth(surface_index=-1) # if len(data.shape) == 4 or (len(data.shape) == 3 and time is None): # from gnome.environment.environment_objects import S_Depth # depth = S_Depth.from_netCDF(grid=grid, # depth=1, # data_file=data_file, # grid_file=grid_file, # **kwargs) if load_all: data = data[:] return cls(name=name, units=units, time=time, data=data, grid=grid, depth=depth, grid_file=grid_file, data_file=data_file, fill_value=fill_value, varname=varname, **kwargs)
def from_netCDF(cls, filename=None, varnames=None, grid_topology=None, name=None, units=None, time=None, grid=None, depth=None, data_file=None, grid_file=None, dataset=None, load_all=False, **kwargs): ''' Allows one-function creation of a GridVectorProp from a file. :param filename: Default data source. Parameters below take precedence :param varnames: Names of the variables in the data source file :param grid_topology: Description of the relationship between grid attributes and variable names. :param name: Name of property :param units: Units :param time: Time axis of the data :param data: Underlying data source :param grid: Grid that the data corresponds with :param dataset: Instance of open Dataset :param data_file: Name of data source file :param grid_file: Name of grid source file :type filename: string :type varnames: [] of string :type grid_topology: {string : string, ...} :type name: string :type units: string :type time: [] of datetime.datetime, netCDF4 Variable, or Time object :type data: netCDF4.Variable or numpy.array :type grid: pysgrid or pyugrid :type dataset: netCDF4.Dataset :type data_file: string :type grid_file: string ''' if filename is not None: data_file = filename grid_file = filename ds = None dg = None if dataset is None: if grid_file == data_file: ds = dg = _get_dataset(grid_file) else: ds = _get_dataset(data_file) dg = _get_dataset(grid_file) else: if grid_file is not None: dg = _get_dataset(grid_file) else: dg = dataset ds = dataset if grid is None: grid = PyGrid.from_netCDF(grid_file, dataset=dg, grid_topology=grid_topology) if varnames is None: varnames = cls._gen_varnames(data_file, dataset=ds) if name is None: name = cls.__name__ + str(cls._def_count) cls._def_count += 1 data = ds[varnames[0]] if time is None: time = Time.from_netCDF(filename=data_file, dataset=ds, datavar=data) if depth is None: if (isinstance(grid, PyGrid_S) and len(data.shape) == 4 or (len(data.shape) == 3 and time is None) or (isinstance(grid, PyGrid_U) and len(data.shape) == 3 or (len(data.shape) == 2 and time is None))): from gnome.environment.environment_objects import Depth depth = Depth(surface_index=-1) # if len(data.shape) == 4 or (len(data.shape) == 3 and time is None): # from gnome.environment.environment_objects import S_Depth # depth = S_Depth.from_netCDF(grid=grid, # depth=1, # data_file=data_file, # grid_file=grid_file, # **kwargs) variables = OrderedCollection(dtype=EnvProp) for vn in varnames: if vn is not None: variables.append( GriddedProp.from_netCDF(filename=filename, varname=vn, grid_topology=grid_topology, units=units, time=time, grid=grid, depth=depth, data_file=data_file, grid_file=grid_file, dataset=ds, load_all=load_all, **kwargs)) if units is None: units = [v.units for v in variables] if all(u == units[0] for u in units): units = units[0] return cls(name=name, filename=filename, varnames=varnames, grid_topology=grid_topology, units=units, time=time, grid=grid, depth=depth, variables=variables, data_file=data_file, grid_file=grid_file, dataset=ds, load_all=load_all, **kwargs)