def from_xarray(dataset): """ Converts an xarray Dataset into a PyGeode Dataset. Parameters ---------- dataset : xarray.Dataset The dataset to be converted. Returns ------- out : pygeode.Dataset An object which can be used with the pygeode package. """ import xarray as xr from pygeode.dataset import Dataset from pygeode.formats.netcdf import dims2axes from pygeode.formats.cfmeta import decode_cf # Encode the axes/variables with CF metadata. out = [] # Loop over each axis and variable, and wrap as a pygeode.Var object. for varname, var in dataset.variables.items(): # Apply a subset of conventions that are relevant to PyGeode. try: var = xr.conventions.maybe_encode_datetime(var) var = xr.conventions.maybe_encode_timedelta(var) except AttributeError: var = xr.coding.times.CFDatetimeCoder().encode(var) var = xr.coding.times.CFTimedeltaCoder().encode(var) try: var = xr.conventions.maybe_encode_string_dtype(var) except AttributeError: pass # Using an older version of xarray (<0.10.0)? out.append(XArray_DataArray(varname, var)) # Wrap all the Var objects into a pygeode.Dataset object. out = Dataset(out, atts=_fix_atts(dataset.attrs)) # Re-construct the axes as pygeode.axis.NamedAxis objects. out = dims2axes(out) # Re-decode the CF metadata on the PyGeode end. # This will get the approperiate axis types for lat, lon, time, etc. out = decode_cf(out) return out
def finalize_open(dataset, dimtypes={}, namemap={}, varlist=[], cfmeta=True): # {{{ from pygeode.formats import cfmeta as cf # Process CF-metadata? if cfmeta is True: # Skip anything that we're going to override in dimtypes # (so we don't get any meaningless warnings or other crap from cfmeta) dataset = cf.decode_cf(dataset, ignore=list(dimtypes.keys())) # Apply custom axis types? if len(dimtypes) > 0: dataset = set_axistypes(dataset, dimtypes) # Keep only specific variables? if len(varlist) > 0: dataset = whitelist(dataset, varlist) # Rename variables? if len(namemap) > 0: # Check both axes and variables dataset = dataset.rename_vars(vardict=namemap) dataset = dataset.rename_axes(axisdict=namemap) return dataset
def finalize_open(dataset, dimtypes = {}, namemap = {}, varlist = [], cfmeta = True): # {{{ from pygeode.formats import cfmeta as cf # Process CF-metadata? if cfmeta is True: # Skip anything that we're going to override in dimtypes # (so we don't get any meaningless warnings or other crap from cfmeta) dataset = cf.decode_cf(dataset, ignore=list(dimtypes.keys())) # Apply custom axis types? if len(dimtypes) > 0: dataset = set_axistypes(dataset, dimtypes) # Keep only specific variables? if len(varlist) > 0: dataset = whitelist(dataset, varlist) # Rename variables? if len(namemap) > 0: # Check both axes and variables dataset = dataset.rename_vars(vardict=namemap) dataset = dataset.rename_axes(axisdict=namemap) return dataset
def open(url): from pygeode.axis import NamedAxis from pygeode.formats.cfmeta import decode_cf from pygeode.dataset import asdataset from warnings import warn warn( "opendap.open is deprecated. Please use netcdf.open, if you have a netcdf library with opendap support compiled in. It is more flexible, faster, and standard.", stacklevel=2) dds = readurl(url + ".dds") # Get list of variables, axes, and types from the url dataset = parse_dataset(tokenize(dds)) # Get metadata das = readurl(url + ".das") attributes = parse_attributes(tokenize(das)) # Get the axes axis_names = [] axis_shapes = [] for (daptype, name, dimnames, shape) in dataset: for dimname, length in zip(dimnames, shape): #TODO: handle unnamed axes assert dimname is not None, "can't handle unnamed axes" if dimname not in axis_names: axis_names.append(dimname) axis_shapes.append(int(length)) # Construct the axes dataset_axes = [] for axis, length in zip(axis_names, axis_shapes): try: values = load_array(url + ".dods?" + axis) # catch any problematic axes # i.e., nbnds in http://test.opendap.org/dap/data/nc/sst.mnmean.nc.gz # (which doesn't have any values!!) except ValueError: from warnings import warn from numpy import arange warn("No values found for axis '%s'" % axis, stacklevel=2) values = arange(length) atts = {} # get the axis attributes for A in attributes: # Match to axis name if A[0] == axis: atts = dict([(k, v) for t, k, v in A[1]]) # Construct this axis dataset_axes.append(NamedAxis(values=values, name=axis, atts=atts)) # Construct the vars dataset_vars = [] for daptype, name, dimnames, shape in dataset: # Don't treat axes as vars if name in axis_names: continue axes = [a for n in dimnames for a in dataset_axes if a.name == n] dtype = dap2np[daptype] var = OpenDAP_Var(name, axes, dtype, url) atts = {} # get the var attributes for A in attributes: # Match to var name if A[0] == name: atts = dict([(k, v) for t, k, v in A[1]]) if len(atts) > 0: var.atts = atts dataset_vars.append(var) # Create a dataset dataset = decode_cf(dataset_vars) # Global attributes? for A in attributes: if A[0] == 'NC_GLOBAL': dataset.atts = dict([(k, v) for t, k, v in A[1]]) return dataset
def open (url): from pygeode.axis import NamedAxis from pygeode.formats.cfmeta import decode_cf from pygeode.dataset import asdataset from warnings import warn warn ("opendap.open is deprecated. Please use netcdf.open, if you have a netcdf library with opendap support compiled in. It is more flexible, faster, and standard.", stacklevel=2) dds = readurl(url+".dds") # Get list of variables, axes, and types from the url dataset = parse_dataset(tokenize(dds)) # Get metadata das = readurl(url+".das") attributes = parse_attributes(tokenize(das)) # Get the axes axis_names = [] axis_shapes = [] for (daptype,name,dimnames,shape) in dataset: for dimname,length in zip(dimnames,shape): #TODO: handle unnamed axes assert dimname is not None, "can't handle unnamed axes" if dimname not in axis_names: axis_names.append(dimname) axis_shapes.append(int(length)) # Construct the axes dataset_axes = [] for axis, length in zip(axis_names, axis_shapes): try: values = load_array(url+".dods?"+axis) # catch any problematic axes # i.e., nbnds in http://test.opendap.org/dap/data/nc/sst.mnmean.nc.gz # (which doesn't have any values!!) except ValueError: from warnings import warn from numpy import arange warn ("No values found for axis '%s'"%axis, stacklevel=2) values = arange(length) atts = {} # get the axis attributes for A in attributes: # Match to axis name if A[0] == axis: atts = dict([(k,v) for t,k,v in A[1]]) # Construct this axis dataset_axes.append(NamedAxis(values=values, name=axis, atts=atts)) # Construct the vars dataset_vars = [] for daptype, name, dimnames, shape in dataset: # Don't treat axes as vars if name in axis_names: continue axes = [a for n in dimnames for a in dataset_axes if a.name == n] dtype = dap2np[daptype] var = OpenDAP_Var(name, axes, dtype, url) atts = {} # get the var attributes for A in attributes: # Match to var name if A[0] == name: atts = dict([(k,v) for t,k,v in A[1]]) if len(atts) > 0: var.atts = atts dataset_vars.append(var) # Create a dataset dataset = decode_cf(dataset_vars) # Global attributes? for A in attributes: if A[0] == 'NC_GLOBAL': dataset.atts = dict([(k,v) for t,k,v in A[1]]) return dataset