def from_xarray(dataset):
"""
Converts an xarray Dataset into a PyGeode Dataset.
Parameters
----------
dataset : xarray.Dataset
The dataset to be converted.
Returns
-------
out : pygeode.Dataset
An object which can be used with the pygeode package.
"""
import xarray as xr
from pygeode.dataset import Dataset
from pygeode.formats.netcdf import dims2axes
from pygeode.formats.cfmeta import decode_cf
# Encode the axes/variables with CF metadata.
out = []
# Loop over each axis and variable, and wrap as a pygeode.Var object.
for varname, var in dataset.variables.items():
# Apply a subset of conventions that are relevant to PyGeode.
try:
var = xr.conventions.maybe_encode_datetime(var)
var = xr.conventions.maybe_encode_timedelta(var)
except AttributeError:
var = xr.coding.times.CFDatetimeCoder().encode(var)
var = xr.coding.times.CFTimedeltaCoder().encode(var)
try:
var = xr.conventions.maybe_encode_string_dtype(var)
except AttributeError:
pass # Using an older version of xarray (<0.10.0)?
out.append(XArray_DataArray(varname, var))
# Wrap all the Var objects into a pygeode.Dataset object.
out = Dataset(out, atts=_fix_atts(dataset.attrs))
# Re-construct the axes as pygeode.axis.NamedAxis objects.
out = dims2axes(out)
# Re-decode the CF metadata on the PyGeode end.
# This will get the approperiate axis types for lat, lon, time, etc.
out = decode_cf(out)
return out
def from_xarray(dataset):
"""
Converts an xarray Dataset into a PyGeode Dataset.
Parameters
----------
dataset : xarray.Dataset
The dataset to be converted.
Returns
-------
out : pygeode.Dataset
An object which can be used with the pygeode package.
"""
import xarray as xr
from pygeode.dataset import Dataset
from pygeode.formats.netcdf import dims2axes
from pygeode.formats.cfmeta import decode_cf
# Encode the axes/variables with CF metadata.
out = []
# Loop over each axis and variable, and wrap as a pygeode.Var object.
for varname, var in dataset.variables.items():
# Apply a subset of conventions that are relevant to PyGeode.
try:
var = xr.conventions.maybe_encode_datetime(var)
var = xr.conventions.maybe_encode_timedelta(var)
except AttributeError:
var = xr.coding.times.CFDatetimeCoder().encode(var)
var = xr.coding.times.CFTimedeltaCoder().encode(var)
try:
var = xr.conventions.maybe_encode_string_dtype(var)
except AttributeError:
pass # Using an older version of xarray (<0.10.0)?
out.append(XArray_DataArray(varname, var))
# Wrap all the Var objects into a pygeode.Dataset object.
out = Dataset(out, atts=_fix_atts(dataset.attrs))
# Re-construct the axes as pygeode.axis.NamedAxis objects.
out = dims2axes(out)
# Re-decode the CF metadata on the PyGeode end.
# This will get the approperiate axis types for lat, lon, time, etc.
out = decode_cf(out)
return out
def finalize_open(dataset, dimtypes={}, namemap={}, varlist=[], cfmeta=True):
# {{{
from pygeode.formats import cfmeta as cf
# Process CF-metadata?
if cfmeta is True:
# Skip anything that we're going to override in dimtypes
# (so we don't get any meaningless warnings or other crap from cfmeta)
dataset = cf.decode_cf(dataset, ignore=list(dimtypes.keys()))
# Apply custom axis types?
if len(dimtypes) > 0:
dataset = set_axistypes(dataset, dimtypes)
# Keep only specific variables?
if len(varlist) > 0:
dataset = whitelist(dataset, varlist)
# Rename variables?
if len(namemap) > 0:
# Check both axes and variables
dataset = dataset.rename_vars(vardict=namemap)
dataset = dataset.rename_axes(axisdict=namemap)
return dataset
def finalize_open(dataset, dimtypes = {}, namemap = {}, varlist = [], cfmeta = True):
# {{{
from pygeode.formats import cfmeta as cf
# Process CF-metadata?
if cfmeta is True:
# Skip anything that we're going to override in dimtypes
# (so we don't get any meaningless warnings or other crap from cfmeta)
dataset = cf.decode_cf(dataset, ignore=list(dimtypes.keys()))
# Apply custom axis types?
if len(dimtypes) > 0:
dataset = set_axistypes(dataset, dimtypes)
# Keep only specific variables?
if len(varlist) > 0:
dataset = whitelist(dataset, varlist)
# Rename variables?
if len(namemap) > 0:
# Check both axes and variables
dataset = dataset.rename_vars(vardict=namemap)
dataset = dataset.rename_axes(axisdict=namemap)
return dataset
def open(url):
from pygeode.axis import NamedAxis
from pygeode.formats.cfmeta import decode_cf
from pygeode.dataset import asdataset
from warnings import warn
warn(
"opendap.open is deprecated. Please use netcdf.open, if you have a netcdf library with opendap support compiled in. It is more flexible, faster, and standard.",
stacklevel=2)
dds = readurl(url + ".dds")
# Get list of variables, axes, and types from the url
dataset = parse_dataset(tokenize(dds))
# Get metadata
das = readurl(url + ".das")
attributes = parse_attributes(tokenize(das))
# Get the axes
axis_names = []
axis_shapes = []
for (daptype, name, dimnames, shape) in dataset:
for dimname, length in zip(dimnames, shape):
#TODO: handle unnamed axes
assert dimname is not None, "can't handle unnamed axes"
if dimname not in axis_names:
axis_names.append(dimname)
axis_shapes.append(int(length))
# Construct the axes
dataset_axes = []
for axis, length in zip(axis_names, axis_shapes):
try:
values = load_array(url + ".dods?" + axis)
# catch any problematic axes
# i.e., nbnds in http://test.opendap.org/dap/data/nc/sst.mnmean.nc.gz
# (which doesn't have any values!!)
except ValueError:
from warnings import warn
from numpy import arange
warn("No values found for axis '%s'" % axis, stacklevel=2)
values = arange(length)
atts = {}
# get the axis attributes
for A in attributes:
# Match to axis name
if A[0] == axis:
atts = dict([(k, v) for t, k, v in A[1]])
# Construct this axis
dataset_axes.append(NamedAxis(values=values, name=axis, atts=atts))
# Construct the vars
dataset_vars = []
for daptype, name, dimnames, shape in dataset:
# Don't treat axes as vars
if name in axis_names: continue
axes = [a for n in dimnames for a in dataset_axes if a.name == n]
dtype = dap2np[daptype]
var = OpenDAP_Var(name, axes, dtype, url)
atts = {}
# get the var attributes
for A in attributes:
# Match to var name
if A[0] == name:
atts = dict([(k, v) for t, k, v in A[1]])
if len(atts) > 0: var.atts = atts
dataset_vars.append(var)
# Create a dataset
dataset = decode_cf(dataset_vars)
# Global attributes?
for A in attributes:
if A[0] == 'NC_GLOBAL':
dataset.atts = dict([(k, v) for t, k, v in A[1]])
return dataset
def open (url):
from pygeode.axis import NamedAxis
from pygeode.formats.cfmeta import decode_cf
from pygeode.dataset import asdataset
from warnings import warn
warn ("opendap.open is deprecated. Please use netcdf.open, if you have a netcdf library with opendap support compiled in. It is more flexible, faster, and standard.", stacklevel=2)
dds = readurl(url+".dds")
# Get list of variables, axes, and types from the url
dataset = parse_dataset(tokenize(dds))
# Get metadata
das = readurl(url+".das")
attributes = parse_attributes(tokenize(das))
# Get the axes
axis_names = []
axis_shapes = []
for (daptype,name,dimnames,shape) in dataset:
for dimname,length in zip(dimnames,shape):
#TODO: handle unnamed axes
assert dimname is not None, "can't handle unnamed axes"
if dimname not in axis_names:
axis_names.append(dimname)
axis_shapes.append(int(length))
# Construct the axes
dataset_axes = []
for axis, length in zip(axis_names, axis_shapes):
try:
values = load_array(url+".dods?"+axis)
# catch any problematic axes
# i.e., nbnds in http://test.opendap.org/dap/data/nc/sst.mnmean.nc.gz
# (which doesn't have any values!!)
except ValueError:
from warnings import warn
from numpy import arange
warn ("No values found for axis '%s'"%axis, stacklevel=2)
values = arange(length)
atts = {}
# get the axis attributes
for A in attributes:
# Match to axis name
if A[0] == axis:
atts = dict([(k,v) for t,k,v in A[1]])
# Construct this axis
dataset_axes.append(NamedAxis(values=values, name=axis, atts=atts))
# Construct the vars
dataset_vars = []
for daptype, name, dimnames, shape in dataset:
# Don't treat axes as vars
if name in axis_names: continue
axes = [a for n in dimnames for a in dataset_axes if a.name == n]
dtype = dap2np[daptype]
var = OpenDAP_Var(name, axes, dtype, url)
atts = {}
# get the var attributes
for A in attributes:
# Match to var name
if A[0] == name:
atts = dict([(k,v) for t,k,v in A[1]])
if len(atts) > 0: var.atts = atts
dataset_vars.append(var)
# Create a dataset
dataset = decode_cf(dataset_vars)
# Global attributes?
for A in attributes:
if A[0] == 'NC_GLOBAL': dataset.atts = dict([(k,v) for t,k,v in A[1]])
return dataset