def test_variable_scatter(self):
var_value = np.arange(5, dtype=float) + 50
var_mask = np.array([True, True, False, True, False])
dest_dist = OcgDist()
five = dest_dist.create_dimension('five', 5, src_idx=np.arange(5), dist=True)
bounds = dest_dist.create_dimension('bounds', 2)
dest_dist.update_dimension_bounds()
if MPI_RANK == 0:
local_dim = Dimension('local', 5, src_idx=np.arange(5))
dim_src_idx = local_dim._src_idx.copy()
var = Variable('the_five', value=var_value, mask=var_mask, dimensions=five.name)
var.set_extrapolated_bounds('the_five_bounds', 'bounds')
var_bounds_value = var.bounds.get_value()
else:
var, var_bounds_value, dim_src_idx = [None] * 3
svar = variable_scatter(var, dest_dist)
var_bounds_value = MPI_COMM.bcast(var_bounds_value)
dim_src_idx = MPI_COMM.bcast(dim_src_idx)
if MPI_RANK > 1:
self.assertIsNone(svar.get_value())
self.assertTrue(svar.is_empty)
else:
dest_dim = dest_dist.get_dimension('five')
self.assertNumpyAll(var_value[slice(*dest_dim.bounds_local)], svar.get_value())
self.assertNumpyAll(var_mask[slice(*dest_dim.bounds_local)], svar.get_mask())
self.assertNumpyAll(var_bounds_value[slice(*dest_dim.bounds_local)], svar.bounds.get_value())
self.assertNumpyAll(dim_src_idx[slice(*dest_dim.bounds_local)], svar.dimensions[0]._src_idx)
self.assertNumpyAll(dim_src_idx[slice(*dest_dim.bounds_local)], svar.bounds.dimensions[0]._src_idx)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel CSV write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.csv')
t = TemporalVariable(name='time',
value=[1, 2, 3],
dtype=float,
dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x',
value=[9, 10, 11, 12],
dimensions='x',
dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y',
value=[13, 14, 15, 16, 17, 18, 19],
dimensions='y',
dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data',
value=np.random.rand(3, 2, 7, 4),
dimensions=['time', 'extra', 'y', 'x'])
vc = VariableCollection(variables=[t, extra, x, y, data])
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path,
iter_kwargs={
'variable': 'data',
'followers': ['time', 'extra', 'y', 'x']
},
driver=DriverCSV)
if MPI_RANK == 0:
desired = 169
with open(path, 'r') as f:
lines = f.readlines()
self.assertEqual(len(lines), desired)
def test_set_extrapolated_bounds(self):
value_grid = [[[40.0, 40.0, 40.0, 40.0], [39.0, 39.0, 39.0, 39.0],
[38.0, 38.0, 38.0, 38.0]],
[[-100.0, -99.0, -98.0, -97.0],
[-100.0, -99.0, -98.0, -97.0],
[-100.0, -99.0, -98.0, -97.0]]]
actual_corners = [[[[40.5, 40.5, 39.5, 39.5], [40.5, 40.5, 39.5, 39.5],
[40.5, 40.5, 39.5, 39.5], [40.5, 40.5, 39.5,
39.5]],
[[39.5, 39.5, 38.5, 38.5], [39.5, 39.5, 38.5, 38.5],
[39.5, 39.5, 38.5, 38.5], [39.5, 39.5, 38.5,
38.5]],
[[38.5, 38.5, 37.5, 37.5], [38.5, 38.5, 37.5, 37.5],
[38.5, 38.5, 37.5, 37.5], [38.5, 38.5, 37.5,
37.5]]],
[[[-100.5, -99.5, -99.5, -100.5],
[-99.5, -98.5, -98.5, -99.5],
[-98.5, -97.5, -97.5, -98.5],
[-97.5, -96.5, -96.5, -97.5]],
[[-100.5, -99.5, -99.5, -100.5],
[-99.5, -98.5, -98.5, -99.5],
[-98.5, -97.5, -97.5, -98.5],
[-97.5, -96.5, -96.5, -97.5]],
[[-100.5, -99.5, -99.5, -100.5],
[-99.5, -98.5, -98.5, -99.5],
[-98.5, -97.5, -97.5, -98.5],
[-97.5, -96.5, -96.5, -97.5]]]]
for should_extrapolate in [False, True]:
y = Variable(name='y',
value=value_grid[0],
dimensions=['ydim', 'xdim'])
x = Variable(name='x',
value=value_grid[1],
dimensions=['ydim', 'xdim'])
if should_extrapolate:
y.set_extrapolated_bounds('ybounds', 'bounds')
x.set_extrapolated_bounds('xbounds', 'bounds')
grid = Grid(x, y)
try:
grid.set_extrapolated_bounds('ybounds', 'xbounds', 'bounds')
except BoundsAlreadyAvailableError:
self.assertTrue(should_extrapolate)
else:
np.testing.assert_equal(grid.y.bounds.get_value(),
actual_corners[0])
np.testing.assert_equal(grid.x.bounds.get_value(),
actual_corners[1])
# Test vectorized.
y = Variable(name='y', value=[1., 2., 3.], dimensions='yy')
x = Variable(name='x', value=[10., 20., 30.], dimensions='xx')
grid = Grid(x, y)
grid.set_extrapolated_bounds('ybounds', 'xbounds', 'bounds')
self.assertEqual(grid.x.bounds.ndim, 2)
self.assertTrue(grid.is_vectorized)
def test_iter_with_bounds(self):
var = Variable(name='bounded', value=[1, 2, 3, 4], dtype=float, dimensions='dim')
var.set_extrapolated_bounds('the_bounds', 'bounds')
lower = Variable(name='lower_bounds', value=var.bounds.get_value()[:, 0], dimensions=var.dimensions)
upper = Variable(name='upper_bounds', value=var.bounds.get_value()[:, 1], dimensions=var.dimensions)
itr = Iterator(var, followers=[lower, upper])
actual = list(itr)
self.assertEqual(len(actual), var.shape[0])
self.assertEqual(len(actual[0]), 3)
def test_init(self):
row = Variable(value=[2, 3], name='row', dimensions='y')
col = Variable(value=[4, 5], name='col', dimensions='x')
grid = Grid(col, row)
self.assertIsNone(grid.archetype.bounds)
row = Variable(value=[2, 3], name='row', dimensions='y')
row.set_extrapolated_bounds('row_bounds', 'bounds')
col = Variable(value=[4, 5], name='col', dimensions='x')
col.set_extrapolated_bounds('col_bounds', 'bounds')
grid = Grid(y=row, x=col)
self.assertEqual(grid.abstraction, 'polygon')
poly = get_geometry_variable(grid)
self.assertEqual(poly.geom_type, 'Polygon')
def test_init(self):
row = Variable(value=[2, 3], name='row', dimensions='y')
col = Variable(value=[4, 5], name='col', dimensions='x')
grid = Grid(col, row)
self.assertIsNone(grid.archetype.bounds)
row = Variable(value=[2, 3], name='row', dimensions='y')
row.set_extrapolated_bounds('row_bounds', 'bounds')
col = Variable(value=[4, 5], name='col', dimensions='x')
col.set_extrapolated_bounds('col_bounds', 'bounds')
grid = Grid(y=row, x=col)
self.assertEqual(grid.abstraction, 'polygon')
poly = get_geometry_variable(grid)
self.assertEqual(poly.geom_type, 'Polygon')
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_expand_grid(self):
x = [101, 102, 103]
y = [40, 41, 42, 43]
vx = Variable('x', value=x, dtype=float, dimensions='xdim')
vx.set_extrapolated_bounds('x_bnds', 'bounds')
vy = Variable('y', value=y, dtype=float, dimensions='ydim')
vy.set_extrapolated_bounds('y_bnds', 'bounds')
grid = Grid(vx, vy)
for variable in [vx, vy]:
self.assertEqual(grid.parent[variable.name].ndim, 1)
expand_grid(grid)
for variable in [vx, vy]:
self.assertEqual(grid.parent[variable.name].ndim, 2)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel CSV write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.csv')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
vc = VariableCollection(variables=[t, extra, x, y, data])
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, iter_kwargs={'variable': 'data', 'followers': ['time', 'extra', 'y', 'x']},
driver=DriverCSV)
if MPI_RANK == 0:
desired = 169
with open(path, 'r') as f:
lines = f.readlines()
self.assertEqual(len(lines), desired)
def test_iter_with_bounds(self):
var = Variable(name='bounded',
value=[1, 2, 3, 4],
dtype=float,
dimensions='dim')
var.set_extrapolated_bounds('the_bounds', 'bounds')
lower = Variable(name='lower_bounds',
value=var.bounds.get_value()[:, 0],
dimensions=var.dimensions)
upper = Variable(name='upper_bounds',
value=var.bounds.get_value()[:, 1],
dimensions=var.dimensions)
itr = Iterator(var, followers=[lower, upper])
actual = list(itr)
self.assertEqual(len(actual), var.shape[0])
self.assertEqual(len(actual[0]), 3)