def test_get_intersects_bounds_sequence(self):
keywords = dict(bounds=[True, False], use_bounds=[True, False])
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# self.log.debug([ctr, k])
# if ctr != 1: continue
# Bounds may not be used if they are not present.
if k.use_bounds and not k.bounds:
continue
y = self.get_variable_y(bounds=k.bounds)
x = self.get_variable_x(bounds=k.bounds)
grid = Grid(x, y)
bounds_sequence = box(-99, 39, -98, 39)
bg = grid.get_intersects(bounds_sequence)
self.assertNotEqual(grid.shape, bg.shape)
self.assertTrue(bg.is_vectorized)
with self.assertRaises(EmptySubsetError):
bounds_sequence = box(1000, 1000, 1001, 10001)
grid.get_intersects(bounds_sequence, use_bounds=k.use_bounds)
bounds_sequence = box(-99999, 1, 1, 1000)
bg2 = grid.get_intersects(bounds_sequence, use_bounds=k.use_bounds)
for target in ['x', 'y']:
original = getattr(grid, target).get_value()
sub = getattr(bg2, target).get_value()
self.assertNumpyAll(original, sub)
# Test mask is not shared with subsetted grid.
grid = self.get_gridxy()
new_mask = grid.get_mask(create=True)
new_mask[:, 1] = True
grid.set_mask(new_mask)
self.assertFalse(grid.has_bounds)
bounds_sequence = box(101.5, 40.5, 103.5, 42.5)
sub = grid.get_intersects(bounds_sequence, use_bounds=False)
new_mask = sub.get_mask()
new_mask.fill(True)
sub.set_mask(new_mask)
self.assertEqual(grid.get_mask().sum(), 4)
def test_get_intersects_masking(self):
"""Test no mask is created if no geometries are masked."""
x = Variable('x', [1, 2], 'x')
y = Variable('y', [1, 2], 'y')
grid = Grid(x, y)
self.assertIsNone(grid.get_mask())
sub = grid.get_intersects(Point(1, 1))
self.assertIsNone(grid.get_mask())
self.assertIsNone(sub.get_mask())
def test_get_intersects_no_slice(self):
"""Test an intersects operations with no slice."""
x = Variable(name='x',
value=[1, 2, 3, 4, 5],
dtype=float,
dimensions='x')
y = Variable(name='y',
value=[1, 2, 3, 4, 5, 6, 7],
dtype=float,
dimensions='y')
grid = Grid(x, y)
subset_geom = Point(3, 4)
sub, the_slice = grid.get_intersects(subset_geom,
return_slice=True,
apply_slice=False)
self.assertEqual(np.sum(np.invert(sub.get_mask())), 1)
self.assertEqual(grid.shape, sub.shape)
self.assertEqual(the_slice, (slice(3, 4, None), slice(2, 3, None)))
sub2 = sub[the_slice]
self.assertEqual(subset_geom,
sub2.get_point().get_value().flatten()[0])
def test_get_intersects_one_rank_with_mask(self):
"""Test mask is created if one rank has a spatial mask."""
if MPI_SIZE != 2:
raise SkipTest('MPI_SIZE != 2')
if MPI_RANK == 0:
value = [1, 2]
else:
value = [3, 4]
ompi = OcgDist()
xdim = ompi.create_dimension('x', 4, dist=True)
ydim = ompi.create_dimension('y', 5, dist=False)
ompi.update_dimension_bounds()
x = Variable('x', value=value, dimensions=xdim)
y = Variable('y', value=[1, 2, 3, 4, 5], dimensions=ydim)
grid = Grid(x, y)
wkt_geom = 'Polygon ((0.72993630573248502 5.22484076433120936, 0.70318471337579691 0.67707006369426814, 2.70063694267515952 0.69490445859872629, 2.59363057324840796 2.54076433121019107, 4.52866242038216527 2.51401273885350296, 4.40382165605095466 5.34968152866241908, 0.72993630573248502 5.22484076433120936))'
subset_geom = wkt.loads(wkt_geom)
sub = grid.get_intersects(subset_geom)
path = self.get_temporary_file_path('foo.nc')
field = Field(grid=sub)
field.write(path)
with vm.scoped('mask count', [0]):
if not vm.is_null:
rd = RequestDataset(path)
out_field = rd.get()
target = out_field[out_field.grid._mask_name].get_value()
select = target != 0
self.assertEqual(select.sum(), 4)
def test_get_intersects_parallel(self):
if sys.version_info.major == 3 and sys.version_info.minor == 5:
raise SkipTest('undefined behavior with Python 3.5')
grid = self.get_gridxy()
live_ranks = vm.get_live_ranks_from_object(grid)
# Test with an empty subset.
subset_geom = box(1000., 1000., 1100., 1100.)
with vm.scoped('empty subset', live_ranks):
if not vm.is_null:
with self.assertRaises(EmptySubsetError):
grid.get_intersects(subset_geom)
# Test combinations.
subset_geom = box(101.5, 40.5, 102.5, 42.)
keywords = dict(is_vectorized=[True, False],
has_bounds=[False, True],
use_bounds=[False, True],
keep_touches=[True, False])
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
grid = self.get_gridxy()
vm_name, _ = vm.create_subcomm_by_emptyable('grid testing',
grid,
is_current=True)
if vm.is_null:
vm.free_subcomm(name=vm_name)
vm.set_comm()
continue
if k.has_bounds:
grid.set_extrapolated_bounds('xbounds', 'ybounds', 'bounds')
self.assertTrue(grid.has_bounds)
# Cannot use bounds with a point grid abstraction.
if k.use_bounds and grid.abstraction == 'point':
vm.free_subcomm(name=vm_name)
vm.set_comm()
continue
grid_sub, slc = grid.get_intersects(subset_geom,
keep_touches=k.keep_touches,
use_bounds=k.use_bounds,
return_slice=True)
if k.has_bounds:
self.assertTrue(grid.has_bounds)
# Test geometries are filled appropriately after allocation.
if not grid_sub.is_empty:
for t in grid_sub.get_abstraction_geometry().get_value().flat:
self.assertIsInstance(t, BaseGeometry)
self.assertIsInstance(grid_sub, Grid)
if k.keep_touches:
if k.has_bounds and k.use_bounds:
desired = (slice(0, 3, None), slice(0, 3, None))
else:
desired = (slice(1, 3, None), slice(1, 2, None))
else:
if k.has_bounds and k.use_bounds:
desired = (slice(1, 3, None), slice(1, 2, None))
else:
desired = (slice(1, 2, None), slice(1, 2, None))
if not grid.is_empty:
self.assertEqual(grid.has_bounds, k.has_bounds)
self.assertTrue(grid.is_vectorized)
self.assertEqual(slc, desired)
vm.free_subcomm(name=vm_name)
vm.set_comm()
# Test against a file. #########################################################################################
subset_geom = box(101.5, 40.5, 102.5, 42.)
if MPI_RANK == 0:
path_grid = self.get_temporary_file_path('grid.nc')
else:
path_grid = None
path_grid = MPI_COMM.bcast(path_grid)
grid_to_write = self.get_gridxy()
with vm.scoped_by_emptyable('write', grid_to_write):
if not vm.is_null:
field = Field(grid=grid_to_write)
field.write(path_grid, driver=DriverNetcdfCF)
MPI_COMM.Barrier()
rd = RequestDataset(uri=path_grid)
x = SourcedVariable(name='x', request_dataset=rd)
self.assertIsNone(x._value)
y = SourcedVariable(name='y', request_dataset=rd)
self.assertIsNone(x._value)
self.assertIsNone(y._value)
grid = Grid(x, y)
for target in [grid._y_name, grid._x_name]:
self.assertIsNone(grid.parent[target]._value)
self.assertTrue(grid.is_vectorized)
with vm.scoped_by_emptyable('intersects', grid):
if not vm.is_null:
sub, slc = grid.get_intersects(subset_geom, return_slice=True)
self.assertEqual(slc, (slice(1, 3, None), slice(1, 2, None)))
self.assertIsInstance(sub, Grid)
# The file may be deleted before other ranks open.
MPI_COMM.Barrier()
def test_get_intersects_ordering(self):
"""Test grid ordering/origins do not influence grid subsetting."""
keywords = {
KeywordArgument.OPTIMIZED_BBOX_SUBSET: [False, True],
'should_wrap': [False, True],
'reverse_x': [False, True],
'reverse_y': [False, True],
'should_expand': [False, True],
}
x_value = np.array(
[155., 160., 165., 170., 175., 180., 185., 190., 195., 200., 205.])
y_value = np.array([-20., -15., -10., -5., 0., 5., 10., 15., 20.])
bbox = [168., -12., 191., 5.3]
for k in self.iter_product_keywords(keywords, as_namedtuple=False):
reverse_x = k.pop('reverse_x')
reverse_y = k.pop('reverse_y')
should_expand = k.pop('should_expand')
should_wrap = k.pop('should_wrap')
ompi = OcgDist()
ompi.create_dimension('dx', len(x_value), dist=True)
ompi.create_dimension('dy', len(y_value))
ompi.update_dimension_bounds()
if reverse_x:
new_x_value = x_value.copy()
new_x_value = np.flipud(new_x_value)
else:
new_x_value = x_value
if reverse_y:
new_y_value = y_value.copy()
new_y_value = np.flipud(new_y_value)
else:
new_y_value = y_value
if MPI_RANK == 0:
x = Variable('x', new_x_value, 'dx')
y = Variable('y', new_y_value, 'dy')
else:
x, y = [None, None]
x = variable_scatter(x, ompi)
y = variable_scatter(y, ompi)
grid = Grid(x, y, crs=Spherical())
with vm.scoped_by_emptyable('scattered', grid):
if not vm.is_null:
if should_expand:
expand_grid(grid)
if should_wrap:
grid = deepcopy(grid)
grid.wrap()
actual_bbox = MultiPolygon([
box(-180, -12, -169, 5.3),
box(168, -12, 180, 5.3)
])
else:
actual_bbox = box(*bbox)
live_ranks = vm.get_live_ranks_from_object(grid)
with vm.scoped('grid.get_intersects', live_ranks):
if not vm.is_null:
sub = grid.get_intersects(actual_bbox, **k)
with vm.scoped_by_emptyable('sub grid', sub):
if not vm.is_null:
if should_wrap:
current_x_value = sub.x.get_value()
current_x_value[
sub.x.get_value() < 0] += 360
self.assertEqual(
sub.extent_global,
(170.0, -10.0, 190.0, 5.0))
if should_expand:
desired = False
else:
desired = True
self.assertEqual(grid.is_vectorized,
desired)
self.assertEqual(sub.is_vectorized,
desired)
self.assertFalse(grid.has_allocated_point)
self.assertFalse(
grid.has_allocated_polygon)