def test_comment(self):
cell_method_strings = [
"time: maximum (interval: 1 hr comment: first bit) "
"time: mean (interval: 1 day comment: second bit)",
"time : maximum (interval: 1 hr comment: first bit) "
"time: mean (interval: 1 day comment: second bit)",
"time: maximum (interval: 1 hr comment: first bit) "
"time : mean (interval: 1 day comment: second bit)",
"time : maximum (interval: 1 hr comment: first bit) "
"time : mean (interval: 1 day comment: second bit)",
]
expected = (
CellMethod(
method="maximum",
coords="time",
intervals="1 hr",
comments="first bit",
),
CellMethod(
method="mean",
coords="time",
intervals="1 day",
comments="second bit",
),
)
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_simple(self):
cell_method_strings = [
'time: mean',
'time : mean',
]
expected = (CellMethod(method='mean', coords='time'), )
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_simple(self):
cell_method_strings = [
"time: mean",
"time : mean",
]
expected = (CellMethod(method="mean", coords="time"),)
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_simple(self):
cell_method_strings = [
'time: mean',
'time : mean',
]
expected = (CellMethod(method='mean', coords='time'),)
for cell_method_str in cell_method_strings:
res = parse_cell_methods('test_var', cell_method_str)
self.assertEqual(res, expected)
def test_portions_of_cells(self):
cell_method_strings = [
'area: mean where sea_ice over sea',
'area : mean where sea_ice over sea',
]
expected = (CellMethod(method='mean where sea_ice over sea',
coords='area'), )
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_with_interval(self):
cell_method_strings = [
'time: variance (interval: 1 hr)',
'time : variance (interval: 1 hr)',
]
expected = (CellMethod(method='variance', coords='time',
intervals='1 hr'),)
for cell_method_str in cell_method_strings:
res = parse_cell_methods('test_var', cell_method_str)
self.assertEqual(res, expected)
def test_portions_of_cells(self):
cell_method_strings = [
'area: mean where sea_ice over sea',
'area : mean where sea_ice over sea',
]
expected = (CellMethod(method='mean where sea_ice over sea',
coords='area'),)
for cell_method_str in cell_method_strings:
res = parse_cell_methods('test_var', cell_method_str)
self.assertEqual(res, expected)
def _load_data(self, array_path, group_dims, grid_mapping,
attr_name=None, separator='__', handle_nan=None):
"""
Create an Iris cube from a TileDB array describing a data variable and
pre-loaded dimension-describing coordinates.
TODO not handled here: aux coords and dims, cell measures, aux factories.
"""
single_attr_name = 'dataset'
if attr_name is None:
attr_metadata, lazy_data = self._from_tdb_array(array_path, single_attr_name,
to_dask=True, handle_nan=handle_nan)
metadata = attr_metadata
attr_name = metadata.pop(single_attr_name)
else:
attr_metadata, lazy_data = self._from_tdb_array(array_path,
single_attr_name,
array_name=attr_name,
to_dask=True,
handle_nan=handle_nan)
metadata = {}
for key, value in attr_metadata.items():
# Varname-specific keys are of form `keyname__attrname`; we only want `keyname`.
# TODO pass the separator character to the method.
try:
key_name, key_attr = key.split(separator)
if key_attr == attr_name:
metadata[key_name] = value
except ValueError:
# Not all keys are varname-specific; we want all of these.
metadata[key] = value
cell_methods = parse_cell_methods(metadata.pop('cell_methods', None))
dim_names = metadata.pop('dimensions').split(',')
# Dim Coords And Dims (mapping of coords to cube axes).
dcad = [(group_dims[name], i) for i, name in enumerate(dim_names)]
safe_attrs = self._handle_attributes(metadata,
exclude_keys=['dataset', 'multiattr', 'grid_mapping'])
std_name = metadata.pop('standard_name', None)
long_name = metadata.pop('long_name', None)
var_name = metadata.pop('var_name', None)
if all(itm is None for itm in [std_name, long_name, var_name]):
long_name = attr_name
cube = Cube(lazy_data,
standard_name=std_name,
long_name=long_name,
var_name=var_name,
units=metadata.pop('units', '1'),
dim_coords_and_dims=dcad,
cell_methods=cell_methods,
attributes=safe_attrs)
cube.coord_system = grid_mapping
return cube
def test_with_interval(self):
cell_method_strings = [
'time: variance (interval: 1 hr)',
'time : variance (interval: 1 hr)',
]
expected = (CellMethod(method='variance',
coords='time',
intervals='1 hr'), )
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_with_interval(self):
cell_method_strings = [
"time: variance (interval: 1 hr)",
"time : variance (interval: 1 hr)",
]
expected = (
CellMethod(method="variance", coords="time", intervals="1 hr"),
)
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_climatology(self):
cell_method_strings = [
'time: minimum within days time: mean over days',
'time : minimum within days time: mean over days',
'time: minimum within days time : mean over days',
'time : minimum within days time : mean over days',
]
expected = (CellMethod(method='minimum within days', coords='time'),
CellMethod(method='mean over days', coords='time'))
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_climatology(self):
cell_method_strings = [
'time: minimum within days time: mean over days',
'time : minimum within days time: mean over days',
'time: minimum within days time : mean over days',
'time : minimum within days time : mean over days',
]
expected = (CellMethod(method='minimum within days', coords='time'),
CellMethod(method='mean over days', coords='time'))
for cell_method_str in cell_method_strings:
res = parse_cell_methods('test_var', cell_method_str)
self.assertEqual(res, expected)
def test_climatology(self):
cell_method_strings = [
"time: minimum within days time: mean over days",
"time : minimum within days time: mean over days",
"time: minimum within days time : mean over days",
"time : minimum within days time : mean over days",
]
expected = (
CellMethod(method="minimum within days", coords="time"),
CellMethod(method="mean over days", coords="time"),
)
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_multiple(self):
cell_method_strings = [
"time: maximum (interval: 1 hr) time: mean (interval: 1 day)",
"time : maximum (interval: 1 hr) time: mean (interval: 1 day)",
"time: maximum (interval: 1 hr) time : mean (interval: 1 day)",
"time : maximum (interval: 1 hr) time : mean (interval: 1 day)",
]
expected = (
CellMethod(method="maximum", coords="time", intervals="1 hr"),
CellMethod(method="mean", coords="time", intervals="1 day"),
)
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_multiple(self):
cell_method_strings = [
'time: maximum (interval: 1 hr) time: mean (interval: 1 day)',
'time : maximum (interval: 1 hr) time: mean (interval: 1 day)',
'time: maximum (interval: 1 hr) time : mean (interval: 1 day)',
'time : maximum (interval: 1 hr) time : mean (interval: 1 day)',
]
expected = (CellMethod(method='maximum', coords='time',
intervals='1 hr'),
CellMethod(method='mean', coords='time',
intervals='1 day'))
for cell_method_str in cell_method_strings:
res = parse_cell_methods('test_var', cell_method_str)
self.assertEqual(res, expected)
def build_cube_metadata(engine):
"""Add the standard meta data to the cube."""
cf_var = engine.cf_var
cube = engine.cube
# Determine the cube's name attributes
cube.var_name = cf_var.cf_name
standard_name = getattr(cf_var, CF_ATTR_STD_NAME, None)
long_name = getattr(cf_var, CF_ATTR_LONG_NAME, None)
cube.long_name = long_name
if standard_name is not None:
try:
cube.standard_name = _get_valid_standard_name(standard_name)
except ValueError:
if cube.long_name is not None:
cube.attributes["invalid_standard_name"] = standard_name
else:
cube.long_name = standard_name
# Determine the cube units.
attr_units = get_attr_units(cf_var, cube.attributes)
cube.units = attr_units
# Incorporate cell methods
nc_att_cell_methods = getattr(cf_var, CF_ATTR_CELL_METHODS, None)
with warnings.catch_warnings(record=True) as warning_records:
cube.cell_methods = parse_cell_methods(nc_att_cell_methods)
# Filter to get the warning we are interested in.
warning_records = [
record for record in warning_records
if issubclass(record.category, UnknownCellMethodWarning)
]
if len(warning_records) > 0:
# Output an enhanced warning message.
warn_record = warning_records[0]
name = "{}".format(cf_var.cf_name)
msg = warn_record.message.args[0]
msg = msg.replace("variable", "variable {!r}".format(name))
warnings.warn(message=msg, category=UnknownCellMethodWarning)
# Set the cube global attributes.
for attr_name, attr_value in cf_var.cf_group.global_attributes.items():
try:
cube.attributes[str(attr_name)] = attr_value
except ValueError as e:
msg = "Skipping global attribute {!r}: {}"
warnings.warn(msg.format(attr_name, str(e)))
def test_climatology_with_unknown_method(self):
cell_method_strings = [
'time: min within days time: mean over days',
'time : min within days time: mean over days',
'time: min within days time : mean over days',
'time : min within days time : mean over days',
]
expected = (CellMethod(method='min within days', coords='time'),
CellMethod(method='mean over days', coords='time'))
for cell_method_str in cell_method_strings:
with mock.patch('warnings.warn') as warn:
res = parse_cell_methods(cell_method_str)
self.assertIn("NetCDF variable contains unknown cell method 'min'",
warn.call_args[0][0])
self.assertEqual(res, expected)
def test_climatology_with_unknown_method(self):
cell_method_strings = [
'time: min within days time: mean over days',
'time : min within days time: mean over days',
'time: min within days time : mean over days',
'time : min within days time : mean over days',
]
expected = (CellMethod(method='min within days', coords='time'),
CellMethod(method='mean over days', coords='time'))
for cell_method_str in cell_method_strings:
with mock.patch('warnings.warn') as warn:
res = parse_cell_methods(cell_method_str)
self.assertIn("NetCDF variable contains unknown cell method 'min'",
warn.call_args[0][0])
self.assertEqual(res, expected)
def test_multiple(self):
cell_method_strings = [
'time: maximum (interval: 1 hr) time: mean (interval: 1 day)',
'time : maximum (interval: 1 hr) time: mean (interval: 1 day)',
'time: maximum (interval: 1 hr) time : mean (interval: 1 day)',
'time : maximum (interval: 1 hr) time : mean (interval: 1 day)',
]
expected = (CellMethod(method='maximum',
coords='time',
intervals='1 hr'),
CellMethod(method='mean', coords='time',
intervals='1 day'))
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
def test_comment(self):
cell_method_strings = [
'time: maximum (interval: 1 hr comment: first bit) '
'time: mean (interval: 1 day comment: second bit)',
'time : maximum (interval: 1 hr comment: first bit) '
'time: mean (interval: 1 day comment: second bit)',
'time: maximum (interval: 1 hr comment: first bit) '
'time : mean (interval: 1 day comment: second bit)',
'time : maximum (interval: 1 hr comment: first bit) '
'time : mean (interval: 1 day comment: second bit)',
]
expected = (CellMethod(method='maximum', coords='time',
intervals='1 hr', comments='first bit'),
CellMethod(method='mean', coords='time',
intervals='1 day', comments='second bit'))
for cell_method_str in cell_method_strings:
res = parse_cell_methods('test_var', cell_method_str)
self.assertEqual(res, expected)
def to_iris(dataarray):
""" Convert a DataArray into a Iris Cube
"""
# Iris not a hard dependency
import iris
from iris.fileformats.netcdf import parse_cell_methods
from xarray.core.pycompat import dask_array_type
dim_coords = []
aux_coords = []
for coord_name in dataarray.coords:
coord = encode(dataarray.coords[coord_name])
coord_args = _get_iris_args(coord.attrs)
coord_args['var_name'] = coord_name
axis = None
if coord.dims:
axis = dataarray.get_axis_num(coord.dims)
if coord_name in dataarray.dims:
iris_coord = iris.coords.DimCoord(coord.values, **coord_args)
dim_coords.append((iris_coord, axis))
else:
iris_coord = iris.coords.AuxCoord(coord.values, **coord_args)
aux_coords.append((iris_coord, axis))
args = _get_iris_args(dataarray.attrs)
args['var_name'] = dataarray.name
args['dim_coords_and_dims'] = dim_coords
args['aux_coords_and_dims'] = aux_coords
if 'cell_methods' in dataarray.attrs:
args['cell_methods'] = \
parse_cell_methods(dataarray.attrs['cell_methods'])
# Create the right type of masked array (should be easier after #1769)
if isinstance(dataarray.data, dask_array_type):
from dask.array import ma as dask_ma
masked_data = dask_ma.masked_invalid(dataarray)
else:
masked_data = np.ma.masked_invalid(dataarray)
cube = iris.cube.Cube(masked_data, **args)
return cube
def to_iris(dataarray):
""" Convert a DataArray into a Iris Cube
"""
# Iris not a hard dependency
import iris
from iris.fileformats.netcdf import parse_cell_methods
from xarray.core.pycompat import dask_array_type
dim_coords = []
aux_coords = []
for coord_name in dataarray.coords:
coord = encode(dataarray.coords[coord_name])
coord_args = _get_iris_args(coord.attrs)
coord_args['var_name'] = coord_name
axis = None
if coord.dims:
axis = dataarray.get_axis_num(coord.dims)
if coord_name in dataarray.dims:
iris_coord = iris.coords.DimCoord(coord.values, **coord_args)
dim_coords.append((iris_coord, axis))
else:
iris_coord = iris.coords.AuxCoord(coord.values, **coord_args)
aux_coords.append((iris_coord, axis))
args = _get_iris_args(dataarray.attrs)
args['var_name'] = dataarray.name
args['dim_coords_and_dims'] = dim_coords
args['aux_coords_and_dims'] = aux_coords
if 'cell_methods' in dataarray.attrs:
args['cell_methods'] = \
parse_cell_methods(dataarray.attrs['cell_methods'])
# Create the right type of masked array (should be easier after #1769)
if isinstance(dataarray.data, dask_array_type):
from dask.array import ma as dask_ma
masked_data = dask_ma.masked_invalid(dataarray)
else:
masked_data = np.ma.masked_invalid(dataarray)
cube = iris.cube.Cube(masked_data, **args)
return cube
def to_iris(dataarray):
""" Convert a DataArray into a Iris Cube
"""
# Iris not a hard dependency
import iris
from iris.fileformats.netcdf import parse_cell_methods
dim_coords = []
aux_coords = []
for coord_name in dataarray.coords:
coord = encode(dataarray.coords[coord_name])
coord_args = _get_iris_args(coord.attrs)
coord_args["var_name"] = coord_name
axis = None
if coord.dims:
axis = dataarray.get_axis_num(coord.dims)
if coord_name in dataarray.dims:
try:
iris_coord = iris.coords.DimCoord(coord.values, **coord_args)
dim_coords.append((iris_coord, axis))
except ValueError:
iris_coord = iris.coords.AuxCoord(coord.values, **coord_args)
aux_coords.append((iris_coord, axis))
else:
iris_coord = iris.coords.AuxCoord(coord.values, **coord_args)
aux_coords.append((iris_coord, axis))
args = _get_iris_args(dataarray.attrs)
args["var_name"] = dataarray.name
args["dim_coords_and_dims"] = dim_coords
args["aux_coords_and_dims"] = aux_coords
if "cell_methods" in dataarray.attrs:
args["cell_methods"] = parse_cell_methods(
dataarray.attrs["cell_methods"])
masked_data = duck_array_ops.masked_invalid(dataarray.data)
cube = iris.cube.Cube(masked_data, **args)
return cube
def to_iris(dataarray):
""" Convert a DataArray into a Iris Cube
"""
# Iris not a hard dependency
import iris
from iris.fileformats.netcdf import parse_cell_methods
dim_coords = []
aux_coords = []
for coord_name in dataarray.coords:
coord = encode(dataarray.coords[coord_name])
coord_args = _get_iris_args(coord.attrs)
coord_args['var_name'] = coord_name
axis = None
if coord.dims:
axis = dataarray.get_axis_num(coord.dims)
if coord_name in dataarray.dims:
try:
iris_coord = iris.coords.DimCoord(coord.values, **coord_args)
dim_coords.append((iris_coord, axis))
except ValueError:
iris_coord = iris.coords.AuxCoord(coord.values, **coord_args)
aux_coords.append((iris_coord, axis))
else:
iris_coord = iris.coords.AuxCoord(coord.values, **coord_args)
aux_coords.append((iris_coord, axis))
args = _get_iris_args(dataarray.attrs)
args['var_name'] = dataarray.name
args['dim_coords_and_dims'] = dim_coords
args['aux_coords_and_dims'] = aux_coords
if 'cell_methods' in dataarray.attrs:
args['cell_methods'] = \
parse_cell_methods(dataarray.attrs['cell_methods'])
masked_data = duck_array_ops.masked_invalid(dataarray.data)
cube = iris.cube.Cube(masked_data, **args)
return cube
def test_comment(self):
cell_method_strings = [
'time: maximum (interval: 1 hr comment: first bit) '
'time: mean (interval: 1 day comment: second bit)',
'time : maximum (interval: 1 hr comment: first bit) '
'time: mean (interval: 1 day comment: second bit)',
'time: maximum (interval: 1 hr comment: first bit) '
'time : mean (interval: 1 day comment: second bit)',
'time : maximum (interval: 1 hr comment: first bit) '
'time : mean (interval: 1 day comment: second bit)',
]
expected = (CellMethod(method='maximum',
coords='time',
intervals='1 hr',
comments='first bit'),
CellMethod(method='mean',
coords='time',
intervals='1 day',
comments='second bit'))
for cell_method_str in cell_method_strings:
res = parse_cell_methods(cell_method_str)
self.assertEqual(res, expected)
