def __iter__(self):
non_iterables = [AbstractRequestObject, dict, Field]
if env.USE_ESMF:
from ocgis.regrid.base import ESMF
non_iterables.append(ESMF.Field)
if isinstance(self._value, tuple(non_iterables)):
to_itr = [self._value]
else:
to_itr = self._value
for uid, element in enumerate(to_itr, start=1):
if isinstance(element, dict):
element = RequestDataset(**element)
if env.USE_ESMF and isinstance(element, ESMF.Field):
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
element = get_ocgis_field_from_esmf_field(element)
try:
element = element.copy()
except AttributeError:
element = copy(element)
if element.uid is None:
element.uid = uid
# TODO: Remove me once the driver does not accept request datasets at initialization.
# Try to change the driver UID.
try:
element.driver.rd.uid = uid
except AttributeError:
# The field driver does not keep a copy of the request dataset.
if hasattr(element.driver, 'rd'):
raise
yield element
def __iter__(self):
non_iterables = [AbstractRequestObject, dict, Field]
if env.USE_ESMF:
from ocgis.regrid.base import ESMF
non_iterables.append(ESMF.Field)
if isinstance(self._value, tuple(non_iterables)):
to_itr = [self._value]
else:
to_itr = self._value
for uid, element in enumerate(to_itr, start=1):
if isinstance(element, dict):
element = RequestDataset(**element)
if env.USE_ESMF and isinstance(element, ESMF.Field):
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
element = get_ocgis_field_from_esmf_field(element)
try:
element = element.copy()
except AttributeError:
element = copy(element)
if element.uid is None:
element.uid = uid
# TODO: Remove me once the driver does not accept request datasets at initialization.
# Try to change the driver UID.
try:
element.driver.rd.uid = uid
except AttributeError:
# The field driver does not keep a copy of the request dataset.
if hasattr(element.driver, 'rd'):
raise
yield element
def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ogrid = create_gridxy_global(crs=Spherical())
ofield = create_exact_field(ogrid, 'foo', ntime=3)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
efield = get_esmf_field_from_ocgis_field(ofield)
self.assertEqual(efield.data.shape, (360, 180, 3))
ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 3)
self.assertNumpyAll(ofield.data_variables[0].get_value(),
ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
self.assertNumpyAll(ofield.time.get_value(),
ofield_actual.time.get_value())
def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ofield = self.get_field(nlevel=0, nrlz=0)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
time = ofield.time.extract()
efield = get_esmf_field_from_ocgis_field(ofield)
ofield_actual = get_ocgis_field_from_esmf_field(
efield,
ofield.data_variables[0].dimensions,
dimension_map=ofield.dimension_map,
time=time)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 2)
self.assertNumpyAll(ofield.time.get_value(),
ofield_actual.time.get_value())
self.assertNumpyAll(ofield.data_variables[0].get_value(),
ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
def test_get_ocgis_field_from_esmf_spatial_only(self):
"""Test with spatial information only."""
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
row = Variable(name='row', value=[5, 6], dimensions='row')
col = Variable(name='col', value=[7, 8], dimensions='col')
grid = Grid(col, row)
ofield = Field(grid=grid, crs=Spherical())
efield = get_esmf_field_from_ocgis_field(ofield)
ofield_actual = get_ocgis_field_from_esmf_field(efield)
self.assertEqual(len(ofield_actual.data_variables), 0)
self.assertNumpyAll(grid.get_value_stacked(), ofield_actual.grid.get_value_stacked())
def test_get_ocgis_field_from_esmpy_spatial_only(self):
"""Test with spatial information only."""
row = VectorDimension(value=[5, 6])
col = VectorDimension(value=[7, 8])
grid = SpatialGridDimension(row=row, col=col)
sdim = SpatialDimension(grid=grid)
field = Field(spatial=sdim)
value = np.random.rand(*field.shape)
variable = Variable(value=value, name="foo")
field.variables.add_variable(variable)
efield = self.get_esmf_field(field=field)
self.assertIsInstance(efield, ESMF.Field)
ofield = get_ocgis_field_from_esmf_field(efield)
for attr in ["realization", "temporal", "level"]:
self.assertIsNone(getattr(ofield, attr))
def test_get_ocgis_field_from_esmf_spatial_only(self):
"""Test with spatial information only."""
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
row = Variable(name='row', value=[5, 6], dimensions='row')
col = Variable(name='col', value=[7, 8], dimensions='col')
grid = Grid(col, row)
ofield = Field(grid=grid, crs=Spherical())
efield = get_esmf_field_from_ocgis_field(ofield)
ofield_actual = get_ocgis_field_from_esmf_field(efield)
self.assertEqual(len(ofield_actual.data_variables), 0)
self.assertNumpyAll(grid.get_value_stacked(),
ofield_actual.grid.get_value_stacked())
def __init__(self, init_value):
if isinstance(init_value, self.__class__):
# Allow the dataset object to be initialized by an instance of itself.
self.__dict__ = init_value.__dict__
else:
if init_value is not None:
if isinstance(init_value, RequestDatasetCollection):
init_value = deepcopy(init_value)
else:
if isinstance(init_value, (RequestDataset, dict, Field)):
itr = [init_value]
elif type(init_value) in [list, tuple]:
itr = init_value
else:
should_raise = True
try:
import ESMF
except ImportError:
# ESMF is not a required library
ocgis_lh('Could not import ESMF library.', level=logging.WARN)
else:
if isinstance(init_value, ESMF.Field):
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
field = get_ocgis_field_from_esmf_field(init_value)
itr = [field]
should_raise = False
if should_raise:
raise DefinitionValidationError(self, 'Type not accepted: {0}'.format(type(init_value)))
rdc = RequestDatasetCollection()
for rd in itr:
if not isinstance(rd, Field):
rd = deepcopy(rd)
try:
rdc.update(rd)
except NoDimensionedVariablesFound:
if rd._name is None:
msg = messages.M2.format(rd.uri)
raise DefinitionValidationError(self, msg)
else:
raise
init_value = rdc
else:
init_value = init_value
super(Dataset, self).__init__(init_value)
def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ogrid = create_gridxy_global(crs=Spherical())
ofield = create_exact_field(ogrid, 'foo', ntime=3)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
efield = get_esmf_field_from_ocgis_field(ofield)
self.assertEqual(efield.data.shape, (360, 180, 3))
ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 3)
self.assertNumpyAll(ofield.data_variables[0].get_value(), ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
self.assertNumpyAll(ofield.time.get_value(), ofield_actual.time.get_value())
def test_get_ocgis_field_from_esmpy_field(self):
np.random.seed(1)
temporal = TemporalDimension(value=[3000.0, 4000.0, 5000.0])
level = VectorDimension(value=[10, 20, 30, 40])
realization = VectorDimension(value=[100, 200])
kwds = dict(
crs=[None, CoordinateReferenceSystem(epsg=4326), Spherical()],
with_mask=[False, True],
with_corners=[False, True],
dimensions=[False, True],
drealization=[False, True],
dtemporal=[False, True],
dlevel=[False, True],
)
for k in self.iter_product_keywords(kwds):
row = VectorDimension(value=[1.0, 2.0])
col = VectorDimension(value=[3.0, 4.0])
if k.with_corners:
row.set_extrapolated_bounds()
col.set_extrapolated_bounds()
value_tmin = np.random.rand(2, 3, 4, 2, 2)
tmin = Variable(value=value_tmin, name="tmin")
variables = VariableCollection([tmin])
grid = SpatialGridDimension(row=row, col=col)
sdim = SpatialDimension(grid=grid, crs=k.crs)
field = Field(variables=variables, spatial=sdim, temporal=temporal, level=level, realization=realization)
if k.with_mask:
mask = np.zeros(value_tmin.shape[-2:], dtype=bool)
mask[0, 1] = True
set_new_value_mask_for_field(field, mask)
sdim.set_mask(mask)
self.assertTrue(tmin.value.mask.any())
self.assertTrue(sdim.get_mask().any())
else:
self.assertFalse(tmin.value.mask.any())
self.assertFalse(sdim.get_mask().any())
coll = SpatialCollection()
coll[1] = {field.name: field}
conv = ESMPyConverter([coll])
efield = conv.write()
if k.dimensions:
dimensions = {}
if k.drealization:
dimensions["realization"] = realization
if k.dtemporal:
dimensions["temporal"] = temporal
if k.dlevel:
dimensions["level"] = level
else:
dimensions = None
ofield = get_ocgis_field_from_esmf_field(efield, crs=k.crs, dimensions=dimensions)
self.assertIsInstance(ofield, Field)
self.assertEqual(ofield.shape, efield.data.shape)
# Test a default CRS is applied for the spherical case.
if k.crs is None:
self.assertEqual(ofield.spatial.crs, Spherical())
if k.drealization and k.dimensions:
target = realization.value
else:
target = np.array([1, 2])
self.assertNumpyAll(ofield.realization.value, target)
if k.dtemporal and k.dimensions:
target = temporal.value
else:
target = np.array([1, 1, 1])
with self.assertRaises(CannotFormatTimeError):
ofield.temporal.value_datetime
self.assertFalse(ofield.temporal.format_time)
self.assertNumpyAll(ofield.temporal.value, target)
if k.dlevel and k.dimensions:
target = level.value
else:
target = np.array([1, 2, 3, 4])
self.assertNumpyAll(ofield.level.value, target)
self.assertNumpyAll(field.spatial.grid.value, ofield.spatial.grid.value)
if k.with_corners:
self.assertIsNotNone(ofield.spatial.grid.corners)
self.assertNumpyAll(field.spatial.grid.corners, ofield.spatial.grid.corners)
try:
self.assertEqual(ofield.spatial.crs, sdim.crs)
except AssertionError:
# A "None" "crs" argument results in a default coordinate system applied to the output OCGIS field.
self.assertIsNone(k.crs)
ofield_tmin_value = ofield.variables[efield.name].value
for arr1, arr2 in itertools.combinations([tmin.value.data, efield.data, ofield_tmin_value.data], r=2):
self.assertNumpyAll(arr1, arr2, check_arr_type=False)
rows = list(ofield.get_iter())
try:
self.assertEqual(len(rows), len(value_tmin.flatten()))
except AssertionError:
self.assertTrue(k.with_mask)
self.assertEqual(len(rows), len(tmin.value.compressed()))
self.assertTrue(np.may_share_memory(ofield_tmin_value, efield.data))
self.assertFalse(np.may_share_memory(ofield_tmin_value, tmin.value))
