def test_set_extrapolated_bounds(self):
value_grid = [[[40.0, 40.0, 40.0, 40.0], [39.0, 39.0, 39.0, 39.0],
[38.0, 38.0, 38.0, 38.0]],
[[-100.0, -99.0, -98.0, -97.0],
[-100.0, -99.0, -98.0, -97.0],
[-100.0, -99.0, -98.0, -97.0]]]
actual_corners = [[[[40.5, 40.5, 39.5, 39.5], [40.5, 40.5, 39.5, 39.5],
[40.5, 40.5, 39.5, 39.5], [40.5, 40.5, 39.5,
39.5]],
[[39.5, 39.5, 38.5, 38.5], [39.5, 39.5, 38.5, 38.5],
[39.5, 39.5, 38.5, 38.5], [39.5, 39.5, 38.5,
38.5]],
[[38.5, 38.5, 37.5, 37.5], [38.5, 38.5, 37.5, 37.5],
[38.5, 38.5, 37.5, 37.5], [38.5, 38.5, 37.5,
37.5]]],
[[[-100.5, -99.5, -99.5, -100.5],
[-99.5, -98.5, -98.5, -99.5],
[-98.5, -97.5, -97.5, -98.5],
[-97.5, -96.5, -96.5, -97.5]],
[[-100.5, -99.5, -99.5, -100.5],
[-99.5, -98.5, -98.5, -99.5],
[-98.5, -97.5, -97.5, -98.5],
[-97.5, -96.5, -96.5, -97.5]],
[[-100.5, -99.5, -99.5, -100.5],
[-99.5, -98.5, -98.5, -99.5],
[-98.5, -97.5, -97.5, -98.5],
[-97.5, -96.5, -96.5, -97.5]]]]
for should_extrapolate in [False, True]:
y = Variable(name='y',
value=value_grid[0],
dimensions=['ydim', 'xdim'])
x = Variable(name='x',
value=value_grid[1],
dimensions=['ydim', 'xdim'])
if should_extrapolate:
y.set_extrapolated_bounds('ybounds', 'bounds')
x.set_extrapolated_bounds('xbounds', 'bounds')
grid = Grid(x, y)
try:
grid.set_extrapolated_bounds('ybounds', 'xbounds', 'bounds')
except BoundsAlreadyAvailableError:
self.assertTrue(should_extrapolate)
else:
np.testing.assert_equal(grid.y.bounds.get_value(),
actual_corners[0])
np.testing.assert_equal(grid.x.bounds.get_value(),
actual_corners[1])
# Test vectorized.
y = Variable(name='y', value=[1., 2., 3.], dimensions='yy')
x = Variable(name='x', value=[10., 20., 30.], dimensions='xx')
grid = Grid(x, y)
grid.set_extrapolated_bounds('ybounds', 'xbounds', 'bounds')
self.assertEqual(grid.x.bounds.ndim, 2)
self.assertTrue(grid.is_vectorized)
def test_init(self):
row = Variable(value=[2, 3], name='row', dimensions='y')
col = Variable(value=[4, 5], name='col', dimensions='x')
grid = Grid(col, row)
self.assertIsNone(grid.archetype.bounds)
row = Variable(value=[2, 3], name='row', dimensions='y')
row.set_extrapolated_bounds('row_bounds', 'bounds')
col = Variable(value=[4, 5], name='col', dimensions='x')
col.set_extrapolated_bounds('col_bounds', 'bounds')
grid = Grid(y=row, x=col)
self.assertEqual(grid.abstraction, 'polygon')
poly = get_geometry_variable(grid)
self.assertEqual(poly.geom_type, 'Polygon')
def test_system_spatial_averaging_through_operations(self):
data_name = 'data'
with vm.scoped('write', [0]):
if not vm.is_null:
x = Variable('x', range(5), 'x', float)
y = Variable('y', range(7), 'y', float)
grid = Grid(x, y)
data_value = np.arange(x.size * y.size).reshape(grid.shape)
data = Variable(data_name, data_value, grid.dimensions, float)
data_value = data.get_value()
field = Field(grid=grid, is_data=data)
path = self.get_temporary_file_path('data.nc')
field.write(path)
else:
data_value, path = None, None
data_value = MPI_COMM.bcast(data_value)
path = MPI_COMM.bcast(path)
rd = RequestDataset(path, variable=data_name)
ops = OcgOperations(dataset=rd, aggregate=True)
ret = ops.execute()
if ret is None:
self.assertNotEqual(vm.rank, vm.root)
else:
out_field = ret.get_element()
if MPI_RANK == 0:
desired = data_value.mean()
actual = out_field.data_variables[0].get_value()[0]
self.assertEqual(actual, desired)
def test_system_get_field_dimensioned_variables(self):
"""Test data is appropriately tagged to identify dimensioned variables."""
path = self.get_temporary_file_path('foo.nc')
time = TemporalVariable(value=[1, 2, 3], dimensions='time')
x = Variable(name='x', value=[10, 20], dimensions='x')
y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y')
data1 = Variable(name='data1',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data2 = Variable(name='data2',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time'])
field = Field(time=time,
grid=Grid(x, y),
variables=[data1, data2, data3])
field.write(path)
# Test dimensioned variables are read from a file with appropriate metadata.
rd = RequestDataset(path)
self.assertEqual(rd.variable, ('data1', 'data2'))
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data1', 'data2'))
# Test dimensioned variables are overloaded.
rd = RequestDataset(path, variable='data2')
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data2', ))
def test_system_process_geometries(self):
"""Test multiple geometries with coordinate system update."""
a = 'POLYGON((-105.21347987288135073 40.21514830508475313,-104.39928495762711691 40.21514830508475313,-104.3192002118643984 39.5677966101694949,-102.37047139830508513 39.61451271186440692,-102.12354343220337682 37.51896186440677639,-105.16009004237288593 37.51896186440677639,-105.21347987288135073 40.21514830508475313))'
b = 'POLYGON((-104.15235699152542281 39.02722457627118757,-103.71189088983049942 39.44099576271186436,-102.71750529661017026 39.28082627118644155,-102.35712394067796538 37.63908898305084705,-104.13900953389830306 37.63241525423728717,-104.15235699152542281 39.02722457627118757))'
geom = [{'geom': wkt.loads(xx), 'properties': {'UGID': ugid}} for ugid, xx in enumerate([a, b])]
grid_value = [
[[37.0, 37.0, 37.0, 37.0], [38.0, 38.0, 38.0, 38.0], [39.0, 39.0, 39.0, 39.0], [40.0, 40.0, 40.0, 40.0]],
[[-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0],
[-105.0, -104.0, -103.0, -102.0]]]
output_crs = CoordinateReferenceSystem(
value={'a': 6370997, 'lon_0': -100, 'y_0': 0, 'no_defs': True, 'proj': 'laea', 'x_0': 0, 'units': 'm',
'b': 6370997, 'lat_0': 45})
x = Variable('x', grid_value[1], dimensions=['lat', 'lon'])
y = Variable('y', grid_value[0], dimensions=['lat', 'lon'])
grid = Grid(x, y)
field = Field(grid=grid, crs=Spherical())
ops = OcgOperations(dataset=field, geom=geom, output_crs=output_crs)
ret = ops.execute()
expected = {0: -502052.79407259845,
1: -510391.37909706926}
for field, container in ret.iter_fields(yield_container=True):
self.assertAlmostEqual(field.grid.get_value_stacked().mean(),
expected[container.geom.ugid.get_value()[0]])
def get_wrap_field(crs=None, unwrapped=True):
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=False)
ompi.create_dimension('y', 7, dist=True)
ompi.create_dimension('time', size_current=4, dist=False)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
row = Variable(value=[-60, -40, -20, 0, 20, 40, 60], name='y', dimensions='y')
if unwrapped:
col_value = [1, 90, 180, 225, 270]
else:
col_value = [-170, -85, 0, 85, 170]
col = Variable(value=col_value, name='x', dimensions='x')
grid = Grid(col, row)
value = np.zeros((4, 7, 5))
for col_idx in range(value.shape[-1]):
value[:, :, col_idx] = col_idx
time = TemporalVariable(name='time', value=[1, 2, 3, 4], dimensions='time')
var = Variable(name='foo', value=value, dimensions=['time', 'y', 'x'])
field = Field(grid=grid, is_data=var, crs=crs, time=time)
else:
field = None
field = variable_collection_scatter(field, ompi)
return field
def test_system_dataset_identifiers_on_variables(self):
"""Test dataset identifiers make it to output variables for iteration."""
paths = []
variables = []
for suffix in [1, 2]:
path = self.get_temporary_file_path('foo{}.nc'.format(suffix))
paths.append(path)
x = Variable(name='x{}'.format(suffix), value=[2, 3], dimensions='x')
y = Variable(name='y{}'.format(suffix), value=[4, 5, 6], dimensions='y')
data_variable_name = 'data{}'.format(suffix)
variables.append(data_variable_name)
data = Variable(name=data_variable_name, value=np.arange(6).reshape(2, 3) + suffix,
dimensions=['x', 'y'])
grid = Grid(x, y)
field = Field(grid=grid, is_data=data)
field.write(path)
rds = [RequestDataset(uri=p, variable=dv) for p, dv in zip(paths, variables)]
ops = OcgOperations(dataset=rds)
rds_uids = [ds.uid for ds in ops.dataset]
self.assertEqual(rds_uids, [1, 2])
ret = ops.execute()
for field in ret.iter_fields():
self.assertFalse(field.grid.has_allocated_abstraction_geometry)
for variable in list(field.values()):
if isinstance(variable, CoordinateReferenceSystem):
continue
self.assertIsNotNone(variable._request_dataset.uid)
for row in variable.get_iter():
self.assertIsNotNone(row[HeaderName.DATASET_IDENTIFER])
def test_get_esmf_grid_with_mask(self):
"""Test with masked data."""
from ocgis.regrid.base import get_esmf_grid
x = Variable(name='x', value=[1, 2, 3], dimensions='x')
y = Variable(name='y', value=[4, 5, 6], dimensions='y')
grid = Grid(x, y, crs=Spherical())
gmask = grid.get_mask(create=True)
gmask[1, 1] = True
grid.set_mask(gmask)
self.assertEqual(grid.get_mask().sum(), 1)
egrid = get_esmf_grid(grid)
egrid_mask_inverted = np.invert(np.array(egrid.mask[0], dtype=bool))
self.assertNumpyAll(grid.get_mask(), egrid_mask_inverted)
# Test with a value mask.
value_mask = np.zeros(grid.shape, dtype=bool)
value_mask[-1, -1] = True
egrid = get_esmf_grid(grid, value_mask=value_mask)
egrid_mask_inverted = np.invert(np.array(egrid.mask[0], dtype=bool))
self.assertNumpyAll(egrid_mask_inverted,
np.logical_or(grid.get_mask(), value_mask))
def get_field(self, ntime=2, variable_name='foo', nrow=2, ncol=2):
"""Create random field where mean varies with radius and std with the
angle around the center of the grid.
"""
np.random.seed(1)
row = Variable(value=np.arange(nrow) - nrow / 2., name='row', dimensions='row')
col = Variable(value=np.arange(ncol) - ncol / 2., name='col', dimensions='col')
grid = Grid(col, row)
x, y = grid.get_value_stacked()
start = dt.datetime(2000, 1, 1)
delta = dt.timedelta(days=1)
value_temporal = [start + i * delta for i in range(ntime)]
temporal = TemporalVariable(value=value_temporal, dimensions='time',
name='time')
nlevel = 1
level = None
nrlz = 1
realization = None
value = np.random.rand(nrlz, ntime, nlevel, nrow, ncol) * np.arctan2(x, y).clip(.1) + np.hypot(x, y)
variable = Variable(name=variable_name,
value=value,
dimensions=['realization', 'time', 'level', 'row',
'col'])
field = Field(grid=grid, time=temporal, is_data=variable, level=level,
realization=realization)
return field
def test_get_mask_from_intersects(self):
poly = wkt.loads(
'POLYGON((-98.26574367088608142 40.19952531645570559,-98.71764240506330168 39.54825949367089066,-99.26257911392406186 39.16281645569620906,-99.43536392405064817 38.64446202531645724,-98.78409810126584034 38.33876582278481493,-98.23916139240508016 37.71408227848101546,-97.77397151898735217 37.67420886075949937,-97.62776898734178133 38.15268987341772799,-98.39865506329114453 38.52484177215190186,-98.23916139240508016 39.33560126582278826,-97.73409810126582897 39.58813291139241386,-97.52143987341773368 40.27927215189873777,-97.52143987341773368 40.27927215189873777,-98.26574367088608142 40.19952531645570559))'
)
desired_mask = np.array([[True, True, False, True],
[True, False, True, True],
[True, True, False, True]])
dist = OcgDist()
xdim = dist.create_dimension('x', 4, dist=True)
ydim = dist.create_dimension('y', 3)
dist.create_dimension('bounds', 2)
dist.update_dimension_bounds()
if MPI_RANK == 0:
x = self.get_variable_x()
y = self.get_variable_y()
grid = Grid(x=x, y=y, abstraction='point', crs=WGS84())
pa = get_geometry_variable(grid)
else:
pa = None
pa = variable_scatter(pa, dist)
vm.create_subcomm_by_emptyable('test_get_mask_from_intersects',
pa,
is_current=True)
if vm.is_null:
self.assertTrue(pa.is_empty)
return
usi = [False]
if env.USE_SPATIAL_INDEX:
usi.append(True)
keywords = dict(use_spatial_index=usi)
for k in self.iter_product_keywords(keywords):
ret = pa.get_mask_from_intersects(
poly, use_spatial_index=k.use_spatial_index)
desired_mask_local = desired_mask[slice(*ydim.bounds_local),
slice(*xdim.bounds_local)]
if MPI_RANK > 1:
self.assertIsNone(ret)
else:
self.assertNumpyAll(desired_mask_local, ret)
# This does not test a parallel operation.
if MPI_RANK == 0:
# Test pre-masked values in geometry are okay for intersects operation.
value = [Point(1, 1), Point(2, 2), Point(3, 3)]
value = np.ma.array(value,
mask=[False, True, False],
dtype=object)
pa2 = GeometryVariable(value=value, dimensions='ngeom')
b = box(0, 0, 5, 5)
res = pa2.get_mask_from_intersects(
b, use_spatial_index=k.use_spatial_index)
self.assertNumpyAll(res, value.mask)
def test_get_intersects_masking(self):
"""Test no mask is created if no geometries are masked."""
x = Variable('x', [1, 2], 'x')
y = Variable('y', [1, 2], 'y')
grid = Grid(x, y)
self.assertIsNone(grid.get_mask())
sub = grid.get_intersects(Point(1, 1))
self.assertIsNone(grid.get_mask())
self.assertIsNone(sub.get_mask())
def test_get_wrapped_state(self):
if sys.version_info.major == 3 and sys.version_info.minor == 5:
raise SkipTest('undefined behavior with Python 3.5')
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=True)
ompi.create_dimension('y', 1)
ompi.update_dimension_bounds()
values = [{
'value': [-179, -90, 0, 90, 180],
'desired': WrappedState.WRAPPED
}, {
'value': [0, 90, 180, 270, 360],
'desired': WrappedState.UNWRAPPED
}, {
'value': [1, 2, 3, 4, 5],
'desired': WrappedState.UNKNOWN
}]
kwds = {'values': values, 'crs': [Spherical(), None]}
for k in self.iter_product_keywords(kwds):
ompi = deepcopy(ompi)
if MPI_RANK == 0:
vx = Variable(name='x',
value=k.values['value'],
dimensions='x')
vy = Variable(name='y', value=[0], dimensions='y')
else:
vx, vy = [None] * 2
vx = variable_scatter(vx, ompi)
vy = variable_scatter(vy, ompi)
grid = Grid(vx, vy)
field = Field(grid=grid, crs=k.crs)
with vm.scoped_by_emptyable('wrap', field):
if not vm.is_null:
wrapped_state = field.wrapped_state
else:
wrapped_state = None
if not field.is_empty:
if k.crs is None:
self.assertIsNone(wrapped_state)
else:
self.assertIsNotNone(wrapped_state)
if k.crs is None or field.is_empty:
self.assertIsNone(wrapped_state)
else:
self.assertEqual(wrapped_state, k.values['desired'])
def test_resolution(self):
for grid in self.get_iter_gridxy():
self.assertEqual(grid.resolution, 1.)
# Test resolution with a singleton dimension.
x = Variable(name='x',
value=[[1, 2, 3, 4]],
dimensions=['first', 'second'])
y = Variable(name='y',
value=[[5, 6, 7, 8]],
dimensions=['first', 'second'])
grid = Grid(x, y)
self.assertEqual(grid.resolution, 1)
def test_get_field_write_target(self):
# Test coordinate system names are added to attributes of dimensioned variables.
x = Variable('x', dimensions='x', value=[1])
y = Variable('y', dimensions='y', value=[2])
t = Variable('t', dimensions='t', value=[3])
crs = WGS84()
d = Variable('data', dimensions=['t', 'y', 'x'], value=[[[1]]])
grid = Grid(x, y)
field = Field(grid=grid, time=t, crs=crs)
field.add_variable(d, is_data=True)
target = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(target[d.name].attrs['grid_mapping'], crs.name)
self.assertEqual(field.x.units, 'degrees_east')
# Test bounds units are removed when writing.
x = Variable(name='x',
value=[1, 2, 3],
dtype=float,
dimensions='x',
units='hours')
y = Variable(name='y',
value=[1, 2, 3],
dtype=float,
dimensions='x',
units='hours')
grid = Grid(x, y)
grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds')
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
field = Field(grid=grid)
actual = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(x.bounds.units, x.units)
self.assertNumpyMayShareMemory(actual[x.name].get_value(),
field[x.name].get_value())
self.assertIsNone(actual[x.name].bounds.units)
self.assertIsNone(actual[y.name].bounds.units)
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
def test_expand_grid(self):
x = [101, 102, 103]
y = [40, 41, 42, 43]
vx = Variable('x', value=x, dtype=float, dimensions='xdim')
vx.set_extrapolated_bounds('x_bnds', 'bounds')
vy = Variable('y', value=y, dtype=float, dimensions='ydim')
vy.set_extrapolated_bounds('y_bnds', 'bounds')
grid = Grid(vx, vy)
for variable in [vx, vy]:
self.assertEqual(grid.parent[variable.name].ndim, 1)
expand_grid(grid)
for variable in [vx, vy]:
self.assertEqual(grid.parent[variable.name].ndim, 2)
def test_get_esmf_grid_periodicity(self):
"""Test periodicity parameters generate reasonable output."""
from ocgis.regrid.base import get_esmf_grid
lon_in = np.arange(-180, 180, 10)
lat_in = np.arange(-90, 90.1, 4)
lon = Variable('lon', lon_in, 'dlon')
lat = Variable('lat', lat_in, 'dlat')
ogrid = Grid(x=lon, y=lat, crs=Spherical())
egrid = get_esmf_grid(ogrid)
self.assertEqual(egrid.periodic_dim, 0)
self.assertEqual(egrid.num_peri_dims, 1)
self.assertEqual(egrid.pole_dim, 1)
def test_get_ocgis_field_from_esmf_spatial_only(self):
"""Test with spatial information only."""
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
row = Variable(name='row', value=[5, 6], dimensions='row')
col = Variable(name='col', value=[7, 8], dimensions='col')
grid = Grid(col, row)
ofield = Field(grid=grid, crs=Spherical())
efield = get_esmf_field_from_ocgis_field(ofield)
ofield_actual = get_ocgis_field_from_esmf_field(efield)
self.assertEqual(len(ofield_actual.data_variables), 0)
self.assertNumpyAll(grid.get_value_stacked(),
ofield_actual.grid.get_value_stacked())
def test_set_extrapolated_bounds_empty(self):
"""Test bounds extrapolation works on empty objects."""
dimx = Dimension('x', 2, is_empty=True, dist=True)
x = Variable('x', dimensions=dimx)
y = Variable('3', dimensions=Dimension('y', 3))
grid = Grid(x, y)
self.assertTrue(dimx.is_empty)
self.assertTrue(x.is_empty)
self.assertTrue(grid.is_empty)
self.assertFalse(grid.has_bounds)
self.assertEqual(grid.abstraction, 'point')
grid.set_extrapolated_bounds('xbnds', 'ybnds', 'bounds')
self.assertEqual(grid.abstraction, 'polygon')
self.assertTrue(grid.has_bounds)
self.assertTrue(grid.is_empty)
def test_get_intersects_bounds_sequence(self):
keywords = dict(bounds=[True, False], use_bounds=[True, False])
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# self.log.debug([ctr, k])
# if ctr != 1: continue
# Bounds may not be used if they are not present.
if k.use_bounds and not k.bounds:
continue
y = self.get_variable_y(bounds=k.bounds)
x = self.get_variable_x(bounds=k.bounds)
grid = Grid(x, y)
bounds_sequence = box(-99, 39, -98, 39)
bg = grid.get_intersects(bounds_sequence)
self.assertNotEqual(grid.shape, bg.shape)
self.assertTrue(bg.is_vectorized)
with self.assertRaises(EmptySubsetError):
bounds_sequence = box(1000, 1000, 1001, 10001)
grid.get_intersects(bounds_sequence, use_bounds=k.use_bounds)
bounds_sequence = box(-99999, 1, 1, 1000)
bg2 = grid.get_intersects(bounds_sequence, use_bounds=k.use_bounds)
for target in ['x', 'y']:
original = getattr(grid, target).get_value()
sub = getattr(bg2, target).get_value()
self.assertNumpyAll(original, sub)
# Test mask is not shared with subsetted grid.
grid = self.get_gridxy()
new_mask = grid.get_mask(create=True)
new_mask[:, 1] = True
grid.set_mask(new_mask)
self.assertFalse(grid.has_bounds)
bounds_sequence = box(101.5, 40.5, 103.5, 42.5)
sub = grid.get_intersects(bounds_sequence, use_bounds=False)
new_mask = sub.get_mask()
new_mask.fill(True)
sub.set_mask(new_mask)
self.assertEqual(grid.get_mask().sum(), 4)
def test_set_mask(self):
grid = self.get_gridxy()
grid.parent['coordinate_system'] = Variable(name='coordinate_system')
self.assertFalse(np.any(grid.get_mask()))
mask = np.zeros(grid.shape, dtype=bool)
mask[1, 1] = True
self.assertTrue(grid.is_vectorized)
grid.set_mask(mask)
self.assertTrue(np.all(grid.get_mask()[1, 1]))
self.assertIn('coordinate_system', grid.parent)
self.assertTrue(grid.is_vectorized)
for mvar in grid.get_member_variables():
if mvar.name != grid._mask_name:
self.assertIsNone(mvar.get_mask())
path = self.get_temporary_file_path('foo.nc')
grid.write(path)
nvc = RequestDataset(path).get()
self.assertIsInstance(nvc, Field)
ngrid = Grid(nvc['x'], nvc['y'], parent=nvc)
# Mask is not written to coordinate variables.
for mvar in ngrid.get_member_variables():
if mvar.name == grid.mask_variable.name:
self.assertTrue(mvar.get_mask()[1, 1])
else:
self.assertIsNone(mvar.get_mask())
# Test with a parent.
grid = self.get_gridxy(with_parent=True)
for k in ['tas', 'rhs']:
self.assertIsNone(grid.parent[k].get_mask())
new_mask = grid.get_mask(create=True)
self.assertFalse(new_mask.any())
new_mask[1:3, 1] = True
grid.set_mask(new_mask, cascade=True)
for k in ['tas', 'rhs']:
backref_var = grid.parent[k]
mask = backref_var.get_mask()
self.assertTrue(mask.any())
if k == 'tas':
self.assertTrue(mask[:, 1, 1:3].all())
if k == 'rhs':
self.assertTrue(mask[1:3, 1, :].all())
self.assertEqual(mask.sum(), 20)
def test_reorder(self):
x = np.linspace(1, 360, num=7, dtype=float)
x = Variable('lon', value=x, dimensions='dimx')
y = Variable('lat', value=[-40, -20, 0, 20, 40], dimensions='dimy')
t = Variable('time', value=[1, 2, 3], dimensions='dimt')
data = Variable(name='data',
value=np.zeros((3, 5, 7)),
dimensions=['dimt', 'dimy', 'dimx'],
dtype=float)
data.get_value()[:] = 10.
data.get_value()[:, :, 3:x.shape[0]] = 20.
data_mask = data.get_mask(create=True)
data_mask[:, :, 3:x.shape[0]] = 1
data.set_mask(data_mask)
parent = Field(variables=[x, y, t, data])
grid = Grid(parent['lon'],
parent['lat'],
crs=Spherical(),
parent=parent)
desired_y = grid.y.get_value().copy()
grid.wrap()
self.assertFalse(np.any(data[:, :, 0:3].get_value() == 20.))
self.assertFalse(np.any(data.get_mask()[:, :, 0:3]))
grid.reorder()
actual = grid.x.get_value().tolist()
desired = [
-179.5, -119.66666666666666, -59.833333333333314, 0.0, 1.0,
60.833333333333336, 120.66666666666667
]
self.assertEqual(actual, desired)
self.assertNumpyAll(desired_y, grid.y.get_value())
rdata = grid.parent['data']
self.assertTrue(np.all(rdata[:, :, 0:3].get_value() == 20.))
self.assertTrue(np.all(rdata.get_mask()[:, :, 0:4]))
self.assertFalse(np.any(rdata.get_mask()[:, :, 4:]))
self.assertNumpyMayShareMemory(rdata.get_value(), data.get_value())
def test_system_masking(self):
"""Test behavior of the grid mask. This is an independently managed variable."""
x = Variable('xc', value=[1, 2, 3], dimensions='dimx')
y = Variable('yc', value=[10, 20, 30, 40], dimensions='dimy')
grid = Grid(x, y)
data = Variable('data',
value=np.zeros(grid.shape),
dimensions=['dimy', 'dimx'])
grid.parent.add_variable(data)
gmask = grid.get_mask()
self.assertIsNone(gmask)
self.assertIsNone(grid.mask_variable)
new_mask = np.zeros(grid.shape, dtype=bool)
new_mask[1, 1] = True
grid.set_mask(new_mask, cascade=True)
self.assertIsInstance(grid.mask_variable, Variable)
actual = grid.get_mask()
self.assertNumpyAll(actual, new_mask)
actual = get_variable_names(grid.get_member_variables())
desired = [x.name, y.name, grid._mask_name]
self.assertAsSetEqual(actual, desired)
self.assertNumpyAll(grid.get_mask(), data.get_mask())
path = self.get_temporary_file_path('foo.nc')
grid.parent.write(path)
with self.nc_scope(path) as ds:
actual = ds.variables[grid.mask_variable.name]
self.assertNumpyAll(grid.get_mask(), actual[:].mask)
# Test mask is used when read from file.
actual_field = RequestDataset(path).get()
self.assertNumpyAll(grid.get_mask(), actual_field.grid.get_mask())
self.assertEqual(actual_field.grid.get_mask().sum(), 1)
self.assertTrue(actual_field.grid.is_vectorized)
self.assertEqual(actual_field.grid.get_mask().dtype, bool)
actual_field.set_abstraction_geom()
self.assertNumpyAll(actual_field.geom.get_mask(), grid.get_mask())
def test_init(self):
# Test a field is always the parent of a grid.
grid = self.get_gridxy()
self.assertIsInstance(grid.parent, Field)
crs = WGS84()
grid = self.get_gridxy(crs=crs)
self.assertIsInstance(grid, Grid)
self.assertIn('x', grid.parent)
self.assertIn('y', grid.parent)
self.assertEqual(grid.crs, crs)
self.assertEqual([dim.name for dim in grid.dimensions],
['ydim', 'xdim'])
self.assertEqual(grid.shape, (4, 3))
self.assertTrue(grid.is_vectorized)
self.assertEqual(grid.x.ndim, 1)
self.assertEqual(grid.y.ndim, 1)
# Test with different variable names.
x = Variable(name='col', value=[1], dimensions='col')
y = Variable(name='row', value=[2], dimensions='row')
grid = Grid(x, y)
assert_equal(grid.x.get_value(), [1])
assert_equal(grid.y.get_value(), [2])
# Test point and polygon representations.
grid = self.get_gridxy(crs=WGS84())
grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds')
targets = ['get_point', 'get_polygon']
targets = [getattr(grid, t)() for t in targets]
for t in targets:
self.assertIsInstance(t, GeometryVariable)
self.assertTrue(grid.is_vectorized)
sub = grid[1, 1]
targets = ['get_point', 'get_polygon']
targets = [getattr(sub, t)() for t in targets]
for t in targets:
self.assertEqual(t.shape, (1, 1))
self.assertIsInstance(t, GeometryVariable)
self.assertTrue(grid.is_vectorized)
def test_get_intersects_no_slice(self):
"""Test an intersects operations with no slice."""
x = Variable(name='x',
value=[1, 2, 3, 4, 5],
dtype=float,
dimensions='x')
y = Variable(name='y',
value=[1, 2, 3, 4, 5, 6, 7],
dtype=float,
dimensions='y')
grid = Grid(x, y)
subset_geom = Point(3, 4)
sub, the_slice = grid.get_intersects(subset_geom,
return_slice=True,
apply_slice=False)
self.assertEqual(np.sum(np.invert(sub.get_mask())), 1)
self.assertEqual(grid.shape, sub.shape)
self.assertEqual(the_slice, (slice(3, 4, None), slice(2, 3, None)))
sub2 = sub[the_slice]
self.assertEqual(subset_geom,
sub2.get_point().get_value().flatten()[0])
def test_init_from_file(self):
"""Test loading from file."""
grid = self.get_gridxy()
path = self.get_temporary_file_path('foo.nc')
grid.write(path)
rd = RequestDataset(uri=path)
x = SourcedVariable(name=grid.x.name,
request_dataset=rd,
protected=True)
y = SourcedVariable(name=grid.y.name,
request_dataset=rd,
protected=True)
self.assertIsNone(x._value)
self.assertIsNone(y._value)
fgrid = Grid(x, y)
self.assertEqual(len(fgrid.dimensions), 2)
for target in [fgrid._y_name, fgrid._x_name]:
fgrid.parent[target].protected = False
actual = np.mean(
[fgrid.x.get_value().mean(),
fgrid.y.get_value().mean()])
self.assertEqual(actual, 71.75)
def test_system_line_subsetting(self):
"""Test subsetting with a line."""
line = LineString([(-0.4, 0.2), (1.35, 0.3), (1.38, -0.716)])
geom = [{'geom': line, 'crs': None}]
x = Variable('x', [-1, -0.5, 0.5, 1.5, 2], 'x')
y = Variable('y', [-0.5, 0.5, 1.5], 'y')
grid = Grid(x, y)
grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds')
field = Field(grid=grid)
ops = OcgOperations(dataset=field, geom=geom)
ret = ops.execute()
field = ret.get_element()
desired = [[[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]], [[-0.5, 0.5, 1.5], [-0.5, 0.5, 1.5]]]
actual = field.grid.get_value_stacked().tolist()
self.assertEqual(actual, desired)
desired = [[True, True, False], [False, False, False]]
actual = field.grid.get_mask().tolist()
self.assertEqual(actual, desired)
def test_system_spherical_unwrapped_grid_negative_values(self):
"""Test with negative values in an unwrapped spherical dataset."""
xc = np.arange(-10, 350, step=10, dtype=float)
yc = np.arange(-90, 100, step=10, dtype=float)
xv = Variable("lon", xc, dimensions=["lon"])
yv = Variable("lat", yc, dimensions=["lat"])
grid = Grid(x=xv, y=yv, crs=Spherical())
self.assertEqual(grid.wrapped_state, WrappedState.UNWRAPPED)
self.assertEqual(grid._wrapped_state, "auto")
subbox = box(240, -10, 360, 10)
subbox = GeometryVariable(name='subset',
value=subbox,
is_bbox=True,
dimensions="ngeom",
crs=Spherical())
so = SpatialSubsetOperation(grid.parent)
subfield = so.get_spatial_subset("intersects", subbox)
self.assertTrue(grid.is_vectorized)
self.assertFalse(subfield.grid.get_mask()[0, 0])
self.assertEqual(grid.x.v()[0], -10.0)
self.assertIsNone(so._transformed_unwrapped_select)
def test_get_distributed_slice(self):
with vm.scoped('grid write', [0]):
if MPI_RANK == 0:
x = Variable('x', list(range(768)), 'x', float)
y = Variable('y', list(range(768)), 'y', float)
grid = Grid(x, y)
field = Field(grid=grid)
path = self.get_temporary_file_path('grid.nc')
field.write(path)
else:
path = None
path = vm.bcast(path)
rd = RequestDataset(path)
grid = rd.get().grid
bounds_global = deepcopy([d.bounds_global for d in grid.dimensions])
for _ in range(10):
_ = grid.get_distributed_slice([slice(73, 157), slice(305, 386)])
bounds_global_grid_after_slice = [
d.bounds_global for d in grid.dimensions
]
self.assertEqual(bounds_global, bounds_global_grid_after_slice)
def test_get_intersects_one_rank_with_mask(self):
"""Test mask is created if one rank has a spatial mask."""
if MPI_SIZE != 2:
raise SkipTest('MPI_SIZE != 2')
if MPI_RANK == 0:
value = [1, 2]
else:
value = [3, 4]
ompi = OcgDist()
xdim = ompi.create_dimension('x', 4, dist=True)
ydim = ompi.create_dimension('y', 5, dist=False)
ompi.update_dimension_bounds()
x = Variable('x', value=value, dimensions=xdim)
y = Variable('y', value=[1, 2, 3, 4, 5], dimensions=ydim)
grid = Grid(x, y)
wkt_geom = 'Polygon ((0.72993630573248502 5.22484076433120936, 0.70318471337579691 0.67707006369426814, 2.70063694267515952 0.69490445859872629, 2.59363057324840796 2.54076433121019107, 4.52866242038216527 2.51401273885350296, 4.40382165605095466 5.34968152866241908, 0.72993630573248502 5.22484076433120936))'
subset_geom = wkt.loads(wkt_geom)
sub = grid.get_intersects(subset_geom)
path = self.get_temporary_file_path('foo.nc')
field = Field(grid=sub)
field.write(path)
with vm.scoped('mask count', [0]):
if not vm.is_null:
rd = RequestDataset(path)
out_field = rd.get()
target = out_field[out_field.grid._mask_name].get_value()
select = target != 0
self.assertEqual(select.sum(), 4)
def test_system_through_operations(self):
"""Test calculation through operations."""
row = Variable(name='y', value=[1, 2, 3, 4], dimensions='y')
col = Variable(name='x', value=[10, 11, 12], dimensions='x')
grid = Grid(col, row)
time = TemporalVariable(name='time', value=[1, 2], dimensions='time')
data = Variable(name='data',
dimensions=[time.dimensions[0]] +
list(grid.dimensions))
data.get_value()[0, :] = 1
data.get_value()[1, :] = 2
field = Field(grid=grid, time=time, is_data=data)
calc = [{'func': 'sum', 'name': 'sum'}]
ops = OcgOperations(dataset=field,
calc=calc,
calc_grouping='day',
calc_raw=True,
aggregate=True)
ret = ops.execute()
actual = ret.get_element(
variable_name='sum').get_masked_value().flatten()
self.assertNumpyAll(actual, np.ma.array([12.0, 24.0]))