def test_create_index_variable_global(self):
raise SkipTest('not implemented')
dsrc_size = 5
ddst_size = 7
dsrc = Dimension('dsrc', dsrc_size, dist=True)
src_dist = OcgDist()
src_dist.add_dimension(dsrc)
src_dist.update_dimension_bounds()
ddst = Dimension('ddst', ddst_size, dist=True)
dst_dist = OcgDist()
dst_dist.add_dimension(ddst)
dst_dist.update_dimension_bounds()
if vm.rank == 0:
np.random.seed(1)
dst = np.random.rand(ddst_size)
src = np.random.choice(dst, size=dsrc_size, replace=False)
src = Variable(name='src', value=src, dimensions=dsrc.name)
# TODO: move create_ugid_global to create_global_index on a standard variable object
dst = GeometryVariable(name='dst', value=dst, dimensions=ddst.name)
else:
src, dst = [None] * 2
src = variable_scatter(src, src_dist)
dst = variable_scatter(dst, dst_dist)
actual = create_index_variable_global('index_array', src, dst)
self.assertNumpyAll(dst.get_value()[actual.get_value()], src.get_value())
def get_wrap_field(crs=None, unwrapped=True):
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=False)
ompi.create_dimension('y', 7, dist=True)
ompi.create_dimension('time', size_current=4, dist=False)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
row = Variable(value=[-60, -40, -20, 0, 20, 40, 60], name='y', dimensions='y')
if unwrapped:
col_value = [1, 90, 180, 225, 270]
else:
col_value = [-170, -85, 0, 85, 170]
col = Variable(value=col_value, name='x', dimensions='x')
grid = Grid(col, row)
value = np.zeros((4, 7, 5))
for col_idx in range(value.shape[-1]):
value[:, :, col_idx] = col_idx
time = TemporalVariable(name='time', value=[1, 2, 3, 4], dimensions='time')
var = Variable(name='foo', value=value, dimensions=['time', 'y', 'x'])
field = Field(grid=grid, is_data=var, crs=crs, time=time)
else:
field = None
field = variable_collection_scatter(field, ompi)
return field
def test_create_unique_global_array(self):
dist = OcgDist()
dist.create_dimension('dim', 9, dist=True)
dist.update_dimension_bounds()
values = [
[4, 2, 1, 2, 1, 4, 1, 4, 2],
[44, 25, 16, 27, 18, 49, 10, 41, 22],
[44, 25, 16, 27, 44, 49, 10, 41, 44],
[1, 1, 1, 1, 1, 1, 1, 1, 1]
]
for v in values:
if vm.rank == 0:
index = Variable(name='cindex', value=v, dimensions='dim')
desired = np.unique(index.get_value())
desired_length = len(desired)
else:
index = None
index = variable_scatter(index, dist)
with vm.scoped_by_emptyable('not empty', index):
if not vm.is_null:
uvar = create_unique_global_array(index.get_value())
uvar_gathered = vm.gather(uvar)
if vm.rank == 0:
uvar_gathered = hgather(uvar_gathered)
self.assertEqual(len(uvar_gathered), desired_length)
self.assertEqual(set(uvar_gathered), set(desired))
def test_write_variable_collection_object_arrays(self):
"""Test writing variable length arrays in parallel."""
with vm.scoped('write', [0]):
if not vm.is_null:
path_actual = self.get_temporary_file_path('in.nc')
path_desired = self.get_temporary_file_path('out.nc')
value = [[1, 3, 5],
[7, 9],
[11]]
v = Variable(name='objects', value=value, fill_value=4, dtype=ObjectType(int), dimensions='values')
v.write(path_desired)
else:
v, path_actual, path_desired = [None] * 3
path_actual = MPI_COMM.bcast(path_actual)
path_desired = MPI_COMM.bcast(path_desired)
dest_mpi = OcgDist()
dest_mpi.create_dimension('values', 3, dist=True)
dest_mpi.update_dimension_bounds()
scattered = variable_scatter(v, dest_mpi)
outvc = VariableCollection(variables=[scattered])
with vm.scoped_by_emptyable('write', outvc):
if not vm.is_null:
outvc.write(path_actual)
if MPI_RANK == 0:
self.assertNcEqual(path_actual, path_desired)
def test_get_distributed_slice_simple(self):
ompi = OcgDist()
dim = ompi.create_dimension('five', 5, dist=True, src_idx='auto')
ompi.update_dimension_bounds(min_elements=1)
with vm.scoped_by_emptyable('simple slice test', dim):
if not vm.is_null:
sub = dim.get_distributed_slice(slice(2, 4))
else:
sub = None
if sub is not None and not sub.is_empty:
self.assertEqual(sub.bounds_global, (0, 2))
else:
if dim.is_empty:
self.assertIsNone(sub)
else:
self.assertEqual(sub.bounds_global, (0, 0))
self.assertEqual(sub.bounds_local, (0, 0))
# Test global bounds are updated.
ompi = OcgDist()
dim = ompi.create_dimension('tester', 768, dist=False)
ompi.update_dimension_bounds()
sub = dim.get_distributed_slice(slice(73, 157))
self.assertEqual(sub.size, 84)
self.assertEqual(sub.bounds_global, (0, 84))
self.assertEqual(sub.bounds_local, (0, 84))
def test_get_unioned_spatial_average_parallel(self):
if MPI_SIZE != 8:
raise SkipTest('MPI_SIZE != 8')
dist = OcgDist()
geom_count = dist.create_dimension('geom_count', size=8, dist=True)
time_count = dist.create_dimension('time', size=3)
dist.update_dimension_bounds()
if not geom_count.is_empty:
gvar = GeometryVariable(value=[Point(1.0, 1.0).buffer(MPI_RANK + 2)] * len(geom_count),
dimensions=geom_count)
value = np.zeros((len(time_count), len(geom_count)), dtype=float)
for ii in range(value.shape[0]):
value[ii] = [MPI_RANK + 1 + ii + 1] * len(geom_count)
data = Variable(name='data', value=value, dtype=float, dimensions=[time_count, geom_count])
else:
gvar = GeometryVariable(dimensions=geom_count)
data = Variable(name='data', dimensions=[time_count, geom_count], dtype=float)
gvar.parent.add_variable(data)
self.assertTrue(gvar.is_empty == data.is_empty == gvar.parent.is_empty)
with vm.scoped_by_emptyable('union', gvar):
if vm.is_null:
unioned = None
else:
unioned = gvar.get_unioned(spatial_average='data')
if unioned is not None:
self.assertIsInstance(unioned, GeometryVariable)
actual = unioned.parent[data.name]
self.assertAlmostEqual(actual.get_value().max(), 5.5466666666666677)
else:
self.assertIsNone(unioned)
def test_init(self):
ompi = OcgDist(size=2)
self.assertEqual(len(ompi.mapping), 2)
dim_x = Dimension('x', 5, dist=False)
dim_y = Dimension('y', 11, dist=True)
var_tas = Variable('tas', value=np.arange(0, 5 * 11).reshape(5, 11), dimensions=(dim_x, dim_y))
thing = Variable('thing', value=np.arange(11) * 10, dimensions=(dim_y,))
vc = VariableCollection(variables=[var_tas, thing])
child = VariableCollection(name='younger')
vc.add_child(child)
childer = VariableCollection(name='youngest')
child.add_child(childer)
dim_six = Dimension('six', 6)
hidden = Variable('hidden', value=[6, 7, 8, 9, 0, 10], dimensions=dim_six)
childer.add_variable(hidden)
ompi.add_dimensions([dim_x, dim_y])
ompi.add_dimension(dim_six, group=hidden.group)
ompi.add_variables([var_tas, thing])
ompi.add_variable(hidden)
var = ompi.get_variable(hidden)
self.assertIsInstance(var, dict)
def test_write_variable_fill_value_is_maintained(self):
if vm.size != 4:
raise SkipTest('vm.size != 4')
dist = OcgDist()
dim = dist.create_dimension('dim', 8, dist=True)
dist.update_dimension_bounds()
var = Variable(name='var', dimensions=dim, fill_value=2.)
var.v()[0] = 1
var.v()[1] = 2
var.get_mask(create=True, check_value=True)
if vm.rank == 0:
path = self.get_temporary_file_path('foo.nc')
else:
path = None
path = vm.bcast(path)
var.parent.write(path)
# if vm.rank == 0:
# self.ncdump(path, header_only=False)
with vm.scoped('read test', [0]):
if not vm.is_null:
invar = RequestDataset(path).create_field()['var']
self.assertEqual(invar.get_mask().sum(), 4)
self.assertEqual(invar.fill_value, 2.)
def test_variable_gather(self):
dist = OcgDist()
three = dist.create_dimension('three', 3, src_idx=np.arange(3) * 10)
four = dist.create_dimension('four', 4, src_idx=np.arange(4, dtype=np.int32), dist=True)
dist.create_variable('four', dimensions=[three, four])
dist.update_dimension_bounds()
if MPI_RANK == 0:
np.random.seed(1)
mask_value = np.random.random(12).reshape(3, 4)
mask = Variable('mask', value=mask_value, dimensions=['three', 'four'])
else:
mask = None
mask = variable_scatter(mask, dist)
with vm.scoped('mask gather', dist.get_empty_ranks(inverse=True)):
if not vm.is_null:
mask_gather = variable_gather(mask)
else:
mask_gather = None
if MPI_RANK == 0:
self.assertNumpyAll(mask_gather.get_value(), mask_value)
self.assertNumpyAll(mask_gather.dimensions[0]._src_idx, np.arange(3) * 10)
self.assertNumpyAll(mask_gather.dimensions[1]._src_idx, np.arange(4, dtype=DataType.DIMENSION_SRC_INDEX))
for dim in mask_gather.dimensions:
self.assertFalse(dim.dist)
else:
self.assertIsNone(mask_gather)
def test_arange_from_dimension(self):
dist = OcgDist()
dim = dist.create_dimension('dim', size=7, dist=True)
dist.update_dimension_bounds()
actual = arange_from_dimension(dim, start=2, dtype=np.int64)
actual = vm.gather(actual)
if vm.rank == 0:
actual = hgather(actual)
desired = np.arange(2, 9, dtype=np.int64)
self.assertNumpyAll(actual, desired)
def test_arange_from_dimension(self):
dist = OcgDist()
dim = dist.create_dimension('dim', size=7, dist=True)
dist.update_dimension_bounds()
actual = arange_from_dimension(dim, start=2, dtype=np.int64)
actual = vm.gather(actual)
if vm.rank == 0:
actual = hgather(actual)
desired = np.arange(2, 9, dtype=np.int64)
self.assertNumpyAll(actual, desired)
def test_variable_gather_bounded_source_index(self):
self.add_barrier = False
dist = OcgDist()
dist.create_dimension('long', 7, dist=True, src_idx='auto')
dist.create_dimension('short', 3, src_idx='auto')
dist.update_dimension_bounds()
if vm.rank == 0:
var = Variable(name='test',
value=np.arange(21).reshape(7, 3),
dimensions=['long', 'short'])
else:
var = None
svar = variable_scatter(var, dist)
with vm.scoped_by_emptyable('gather test', svar):
if vm.is_null:
return
gvar = variable_gather(svar)
if vm.rank == 0:
actual = gvar.dimensions[0]._src_idx
desired = (0, 7)
self.assertEqual(actual, desired)
else:
self.assertIsNone(gvar)
def test_get_distributed_slice_simple(self):
ompi = OcgDist()
dim = ompi.create_dimension('five', 5, dist=True, src_idx='auto')
ompi.update_dimension_bounds(min_elements=1)
with vm.scoped_by_emptyable('simple slice test', dim):
if not vm.is_null:
sub = dim.get_distributed_slice(slice(2, 4))
else:
sub = None
if sub is not None and not sub.is_empty:
self.assertEqual(sub.bounds_global, (0, 2))
else:
if dim.is_empty:
self.assertIsNone(sub)
else:
self.assertEqual(sub.bounds_global, (0, 0))
self.assertEqual(sub.bounds_local, (0, 0))
# Test global bounds are updated.
ompi = OcgDist()
dim = ompi.create_dimension('tester', 768, dist=False)
ompi.update_dimension_bounds()
sub = dim.get_distributed_slice(slice(73, 157))
self.assertEqual(sub.size, 84)
self.assertEqual(sub.bounds_global, (0, 84))
self.assertEqual(sub.bounds_local, (0, 84))
def test_system_grid_chunking(self):
if vm.size != 4: raise SkipTest('vm.size != 4')
from ocgis.spatial.grid_chunker import GridChunker
path = self.path_esmf_unstruct
rd_dst = RequestDataset(uri=path,
driver=DriverESMFUnstruct,
crs=Spherical(),
grid_abstraction='point',
grid_is_isomorphic=True)
rd_src = deepcopy(rd_dst)
resolution = 0.28125
chunk_wd = os.path.join(self.current_dir_output, 'chunks')
if vm.rank == 0:
os.mkdir(chunk_wd)
vm.barrier()
paths = {'wd': chunk_wd}
gc = GridChunker(rd_src, rd_dst, nchunks_dst=[8], src_grid_resolution=resolution,
dst_grid_resolution=resolution,
optimized_bbox_subset=True, paths=paths, genweights=True)
gc.write_chunks()
dist = OcgDist()
local_ctr = Dimension(name='ctr', size=8, dist=True)
dist.add_dimension(local_ctr)
dist.update_dimension_bounds()
for ctr in range(local_ctr.bounds_local[0], local_ctr.bounds_local[1]):
ctr += 1
s = os.path.join(chunk_wd, 'split_src_{}.nc'.format(ctr))
d = os.path.join(chunk_wd, 'split_dst_{}.nc'.format(ctr))
sf = Field.read(s, driver=DriverESMFUnstruct)
df = Field.read(d, driver=DriverESMFUnstruct)
self.assertLessEqual(sf.grid.shape[0] - df.grid.shape[0], 150)
self.assertGreater(sf.grid.shape[0], df.grid.shape[0])
wgt = os.path.join(chunk_wd, 'esmf_weights_{}.nc'.format(ctr))
f = Field.read(wgt)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
with vm.scoped('merge weights', [0]):
if not vm.is_null:
merged_weights = self.get_temporary_file_path('merged_weights.nc')
gc.create_merged_weight_file(merged_weights, strict=False)
f = Field.read(merged_weights)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
def test_get_distributed_slice(self):
self.add_barrier = False
for d in [True, False]:
dist = OcgDist()
dim = dist.create_dimension('five', 5, dist=d, src_idx='auto')
dist.update_dimension_bounds()
if not dim.is_empty:
self.assertEqual(dim.bounds_global, (0, 5))
if dim.dist:
if MPI_RANK > 1:
self.assertTrue(dim.is_empty)
else:
self.assertFalse(dim.is_empty)
with vm.scoped_by_emptyable('dim slice', dim):
if not vm.is_null:
sub = dim.get_distributed_slice(slice(1, 3))
else:
sub = None
if dim.dist:
if not dim.is_empty:
self.assertIsNotNone(dim._src_idx)
else:
self.assertIsNone(sub)
if MPI_SIZE == 2:
desired_emptiness = {0: False, 1: True}[MPI_RANK]
desired_bounds_local = {0: (0, 2), 1: (0, 0)}[MPI_RANK]
self.assertEqual(sub.is_empty, desired_emptiness)
self.assertEqual(sub.bounds_local, desired_bounds_local)
if MPI_SIZE >= 5 and 0 < MPI_RANK > 2:
self.assertTrue(sub is None or sub.is_empty)
else:
self.assertEqual(len(dim), 5)
self.assertEqual(dim.bounds_global, (0, 5))
self.assertEqual(dim.bounds_local, (0, 5))
dist = OcgDist()
dim = dist.create_dimension('five', 5, dist=True, src_idx='auto')
dist.update_dimension_bounds()
with vm.scoped_by_emptyable('five slice', dim):
if not vm.is_null:
sub2 = dim.get_distributed_slice(slice(2, 4))
else:
sub2 = None
if MPI_SIZE == 3 and MPI_RANK == 2:
self.assertIsNone(sub2)
def test_create_ugid_global(self):
ompi = OcgDist()
m = ompi.create_dimension('m', 4)
n = ompi.create_dimension('n', 70, dist=True)
ompi.update_dimension_bounds()
gvar = GeometryVariable(name='geom', dimensions=(m, n))
ugid = gvar.create_ugid_global('gid')
if not gvar.is_empty:
self.assertEqual(gvar.dist, ugid.dist)
gathered = variable_gather(ugid)
if MPI_RANK == 0:
actual = gathered.get_value()
self.assertEqual(actual.size, len(set(actual.flatten().tolist())))
def test_get_mask_from_intersects(self):
poly = wkt.loads(
'POLYGON((-98.26574367088608142 40.19952531645570559,-98.71764240506330168 39.54825949367089066,-99.26257911392406186 39.16281645569620906,-99.43536392405064817 38.64446202531645724,-98.78409810126584034 38.33876582278481493,-98.23916139240508016 37.71408227848101546,-97.77397151898735217 37.67420886075949937,-97.62776898734178133 38.15268987341772799,-98.39865506329114453 38.52484177215190186,-98.23916139240508016 39.33560126582278826,-97.73409810126582897 39.58813291139241386,-97.52143987341773368 40.27927215189873777,-97.52143987341773368 40.27927215189873777,-98.26574367088608142 40.19952531645570559))'
)
desired_mask = np.array([[True, True, False, True],
[True, False, True, True],
[True, True, False, True]])
dist = OcgDist()
xdim = dist.create_dimension('x', 4, dist=True)
ydim = dist.create_dimension('y', 3)
dist.create_dimension('bounds', 2)
dist.update_dimension_bounds()
if MPI_RANK == 0:
x = self.get_variable_x()
y = self.get_variable_y()
grid = Grid(x=x, y=y, abstraction='point', crs=WGS84())
pa = get_geometry_variable(grid)
else:
pa = None
pa = variable_scatter(pa, dist)
vm.create_subcomm_by_emptyable('test_get_mask_from_intersects',
pa,
is_current=True)
if vm.is_null:
self.assertTrue(pa.is_empty)
return
usi = [False]
if env.USE_SPATIAL_INDEX:
usi.append(True)
keywords = dict(use_spatial_index=usi)
for k in self.iter_product_keywords(keywords):
ret = pa.get_mask_from_intersects(
poly, use_spatial_index=k.use_spatial_index)
desired_mask_local = desired_mask[slice(*ydim.bounds_local),
slice(*xdim.bounds_local)]
if MPI_RANK > 1:
self.assertIsNone(ret)
else:
self.assertNumpyAll(desired_mask_local, ret)
# This does not test a parallel operation.
if MPI_RANK == 0:
# Test pre-masked values in geometry are okay for intersects operation.
value = [Point(1, 1), Point(2, 2), Point(3, 3)]
value = np.ma.array(value,
mask=[False, True, False],
dtype=object)
pa2 = GeometryVariable(value=value, dimensions='ngeom')
b = box(0, 0, 5, 5)
res = pa2.get_mask_from_intersects(
b, use_spatial_index=k.use_spatial_index)
self.assertNumpyAll(res, value.mask)
def test_reduce_global(self):
pt = self.fixture(cindex=self.fixture_cindex(1), start_index=1)
self.assertEqual(pt.start_index, 1)
dist = OcgDist()
for d in pt.parent.dimensions.values():
d = d.copy()
if d.name == self.fixture_element_dimension.name:
d.dist = True
dist.add_dimension(d)
dist.update_dimension_bounds()
new_parent = variable_collection_scatter(pt.parent, dist)
vm.create_subcomm_by_emptyable('coordinate reduction', new_parent, is_current=True)
if vm.is_null:
return
pt.parent = new_parent
sub = pt.get_distributed_slice(slice(2, 5))
vm.create_subcomm_by_emptyable('distributed slice', sub, is_current=True)
if vm.is_null:
return
actual = sub.reduce_global()
actual_cindex = actual.cindex.extract()
actual_cindex = variable_gather(actual_cindex)
if vm.rank == 0:
actual_cindex = actual_cindex.get_value().flatten().tolist()
self.assertEqual(actual_cindex, [1, 2, 3])
gathered = [variable_gather(c.extract()) for c in actual.coordinate_variables]
if vm.rank == 0:
actual_coords = []
for c in gathered:
actual_coords.append(c.get_value().tolist())
desired = [[2.0, 3.0, 4.0], [8.0, 9.0, 10.0], [14.0, 15.0, 16.0]]
self.assertEqual(actual_coords, desired)
path = self.get_temporary_file_path('foo.nc')
actual.parent.write(path)
actual = Field.read(path)
self.assertEqual(actual['cindex'].attrs['start_index'], 1)
def test_get_distributed_slice_fancy_indexing(self):
if vm.size != 2:
raise SkipTest('vm.size != 2')
dist = OcgDist()
dim = dist.create_dimension('dim', size=5, dist=True, src_idx='auto')
dist.update_dimension_bounds()
slices = {0: [False, True, True],
1: [False, True]}
slc = slices[vm.rank]
sub = dim.get_distributed_slice(slc)
desired_bounds_global = (0, sum([np.array(ii).sum() for ii in slices.values()]))
desired_bounds_local = {0: (0, 2), 1: (2, 3)}
self.assertEqual(sub.bounds_global, desired_bounds_global)
self.assertEqual(sub.bounds_local, desired_bounds_local[vm.rank])
def test_variable_scatter(self):
var_value = np.arange(5, dtype=float) + 50
var_mask = np.array([True, True, False, True, False])
dest_dist = OcgDist()
five = dest_dist.create_dimension('five', 5, src_idx=np.arange(5), dist=True)
bounds = dest_dist.create_dimension('bounds', 2)
dest_dist.update_dimension_bounds()
if MPI_RANK == 0:
local_dim = Dimension('local', 5, src_idx=np.arange(5))
dim_src_idx = local_dim._src_idx.copy()
var = Variable('the_five', value=var_value, mask=var_mask, dimensions=five.name)
var.set_extrapolated_bounds('the_five_bounds', 'bounds')
var_bounds_value = var.bounds.get_value()
else:
var, var_bounds_value, dim_src_idx = [None] * 3
svar = variable_scatter(var, dest_dist)
var_bounds_value = MPI_COMM.bcast(var_bounds_value)
dim_src_idx = MPI_COMM.bcast(dim_src_idx)
if MPI_RANK > 1:
self.assertIsNone(svar.get_value())
self.assertTrue(svar.is_empty)
else:
dest_dim = dest_dist.get_dimension('five')
self.assertNumpyAll(var_value[slice(*dest_dim.bounds_local)], svar.get_value())
self.assertNumpyAll(var_mask[slice(*dest_dim.bounds_local)], svar.get_mask())
self.assertNumpyAll(var_bounds_value[slice(*dest_dim.bounds_local)], svar.bounds.get_value())
self.assertNumpyAll(dim_src_idx[slice(*dest_dim.bounds_local)], svar.dimensions[0]._src_idx)
self.assertNumpyAll(dim_src_idx[slice(*dest_dim.bounds_local)], svar.bounds.dimensions[0]._src_idx)
def test_get_distributed_slice_uneven_boundary(self):
ompi = OcgDist()
dim = ompi.create_dimension('the_dim', 360, dist=True, src_idx='auto')
ompi.update_dimension_bounds()
sub = dim.get_distributed_slice(slice(12, 112))
if sub.is_empty:
self.assertEqual(sub.bounds_local, (0, 0))
self.assertEqual(sub.bounds_global, (0, 0))
self.assertIsNone(sub._src_idx)
else:
self.assertIsNotNone(sub._src_idx)
self.assertEqual(sub.bounds_global, (0, 100))
if MPI_SIZE == 4:
desired = {0: (0, 78), 1: (78, 100)}
desired = desired.get(MPI_RANK, (0, 0))
self.assertEqual(sub.bounds_local, desired)
def test_get_distributed_slice_uneven_boundary(self):
ompi = OcgDist()
dim = ompi.create_dimension('the_dim', 360, dist=True, src_idx='auto')
ompi.update_dimension_bounds()
sub = dim.get_distributed_slice(slice(12, 112))
if sub.is_empty:
self.assertEqual(sub.bounds_local, (0, 0))
self.assertEqual(sub.bounds_global, (0, 0))
self.assertIsNone(sub._src_idx)
else:
self.assertIsNotNone(sub._src_idx)
self.assertEqual(sub.bounds_global, (0, 100))
if MPI_SIZE == 4:
desired = {0: (0, 78), 1: (78, 100)}
desired = desired.get(MPI_RANK, (0, 0))
self.assertEqual(sub.bounds_local, desired)
def test_reduce_reindex_coordinate_index(self):
dist = OcgDist()
dist.create_dimension('dim', 12, dist=True)
dist.update_dimension_bounds()
global_cindex_arr = np.array([4, 2, 1, 2, 1, 4, 1, 4, 2, 5, 6, 7])
if vm.rank == 0:
var_cindex = Variable('cindex', value=global_cindex_arr, dimensions='dim')
else:
var_cindex = None
var_cindex = variable_scatter(var_cindex, dist)
vm.create_subcomm_by_emptyable('test', var_cindex, is_current=True)
if vm.is_null:
return
raise_if_empty(var_cindex)
coords = np.array([0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 100, 110, 120, 130, 140, 150])
coords = Variable(name='coords', value=coords, dimensions='coord_dim')
new_cindex, u_indices = reduce_reindex_coordinate_index(var_cindex)
desired = coords[global_cindex_arr].get_value()
if len(u_indices) > 0:
new_coords = coords[u_indices].get_value()
else:
new_coords = np.array([])
gathered_new_coords = vm.gather(new_coords)
gathered_new_cindex = vm.gather(new_cindex)
if vm.rank == 0:
gathered_new_coords = hgather(gathered_new_coords)
gathered_new_cindex = hgather(gathered_new_cindex)
actual = gathered_new_coords[gathered_new_cindex]
self.assertAsSetEqual(gathered_new_cindex.tolist(), [2, 1, 0, 3, 4, 5])
desired_new_coords = [11, 22, 44, 55, 66, 77]
self.assertAsSetEqual(gathered_new_coords.tolist(), desired_new_coords)
self.assertEqual(len(gathered_new_coords), len(desired_new_coords))
self.assertNumpyAll(actual, desired)
def create_dist(self, metadata=None):
"""
Create a distribution from global metadata. In general, this should not be overloaded by subclasses.
:param dict metadata: Global metadata to use for creating a distribution.
:rtype: :class:`ocgis.OcgDist`
"""
ompi = OcgDist(size=vm.size, ranks=vm.ranks)
# Convert metadata into a grouping consistent with the MPI dimensions.
if metadata is None:
metadata = self.metadata_source
metadata = {None: metadata}
for group_index in iter_all_group_keys(metadata):
group_meta = get_group(metadata, group_index)
# Add the dimensions to the distribution object.
dimensions = self.create_dimensions(group_meta)
# Only build a distribution if the group has more than one dimension.
if len(dimensions) == 0:
_ = ompi.get_group(group=group_index)
else:
for dimension_name, dimension_meta in list(
group_meta['dimensions'].items()):
target_dimension = dimensions[dimension_name]
target_dimension.dist = group_meta['dimensions'][
dimension_name].get('dist', False)
ompi.add_dimension(target_dimension, group=group_index)
try:
dimension_map = self.rd.dimension_map.get_group(
group_index)
except DimensionMapError:
# Likely a user-provided dimension map.
continue
# dimension_map = get_group(self.rd.dimension_map, group_index, has_root=False)
distributed_dimension_name = self.get_distributed_dimension_name(
dimension_map,
group_meta['dimensions'],
decomp_type=self.rd.decomp_type)
# Allow no distributed dimensions to be returned.
if distributed_dimension_name is not None:
for target_rank in range(ompi.size):
distributed_dimension = ompi.get_dimension(
distributed_dimension_name,
group=group_index,
rank=target_rank)
distributed_dimension.dist = True
ompi.update_dimension_bounds()
return ompi
def test_get_wrapped_state(self):
if sys.version_info.major == 3 and sys.version_info.minor == 5:
raise SkipTest('undefined behavior with Python 3.5')
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=True)
ompi.create_dimension('y', 1)
ompi.update_dimension_bounds()
values = [{
'value': [-179, -90, 0, 90, 180],
'desired': WrappedState.WRAPPED
}, {
'value': [0, 90, 180, 270, 360],
'desired': WrappedState.UNWRAPPED
}, {
'value': [1, 2, 3, 4, 5],
'desired': WrappedState.UNKNOWN
}]
kwds = {'values': values, 'crs': [Spherical(), None]}
for k in self.iter_product_keywords(kwds):
ompi = deepcopy(ompi)
if MPI_RANK == 0:
vx = Variable(name='x',
value=k.values['value'],
dimensions='x')
vy = Variable(name='y', value=[0], dimensions='y')
else:
vx, vy = [None] * 2
vx = variable_scatter(vx, ompi)
vy = variable_scatter(vy, ompi)
grid = Grid(vx, vy)
field = Field(grid=grid, crs=k.crs)
with vm.scoped_by_emptyable('wrap', field):
if not vm.is_null:
wrapped_state = field.wrapped_state
else:
wrapped_state = None
if not field.is_empty:
if k.crs is None:
self.assertIsNone(wrapped_state)
else:
self.assertIsNotNone(wrapped_state)
if k.crs is None or field.is_empty:
self.assertIsNone(wrapped_state)
else:
self.assertEqual(wrapped_state, k.values['desired'])
def create_slices_for_dimension(size, splits):
ompi = OcgDist(size=splits)
dimname = 'foo'
ompi.create_dimension(dimname, size, dist=True)
ompi.update_dimension_bounds()
slices = []
for rank in range(splits):
dimension = ompi.get_dimension(dimname, rank=rank)
slices.append(dimension.bounds_local)
return slices
def test_variable_scatter_ndimensions(self):
r = Dimension('realization', 3)
t = Dimension('time', 365)
l = Dimension('level', 10)
y = Dimension('y', 90, dist=True)
x = Dimension('x', 360, dist=True)
dimensions = [r, t, l, y, x]
dest_mpi = OcgDist()
for d in dimensions:
dest_mpi.add_dimension(d)
dest_mpi.update_dimension_bounds()
if MPI_RANK == 0:
local_dimensions = deepcopy(dimensions)
for l in local_dimensions:
l.dist = False
var = Variable('tas', dimensions=local_dimensions)
else:
var = None
svar = variable_scatter(var, dest_mpi)
self.assertTrue(svar.dist)
self.assertIsNotNone(svar)
if MPI_SIZE == 2:
self.assertEqual(svar.shape, (3, 365, 10, 90, 180))
def test_create_unique_global_array(self):
dist = OcgDist()
dist.create_dimension('dim', 9, dist=True)
dist.update_dimension_bounds()
values = [[4, 2, 1, 2, 1, 4, 1, 4, 2],
[44, 25, 16, 27, 18, 49, 10, 41, 22],
[44, 25, 16, 27, 44, 49, 10, 41, 44],
[1, 1, 1, 1, 1, 1, 1, 1, 1]]
for v in values:
if vm.rank == 0:
index = Variable(name='cindex', value=v, dimensions='dim')
desired = np.unique(index.get_value())
desired_length = len(desired)
else:
index = None
index = variable_scatter(index, dist)
with vm.scoped_by_emptyable('not empty', index):
if not vm.is_null:
uvar = create_unique_global_array(index.get_value())
uvar_gathered = vm.gather(uvar)
if vm.rank == 0:
uvar_gathered = hgather(uvar_gathered)
self.assertEqual(len(uvar_gathered), desired_length)
self.assertEqual(set(uvar_gathered), set(desired))
def test_write_variable_collection_object_arrays(self):
"""Test writing variable length arrays in parallel."""
with vm.scoped('write', [0]):
if not vm.is_null:
path_actual = self.get_temporary_file_path('in.nc')
path_desired = self.get_temporary_file_path('out.nc')
value = [[1, 3, 5], [7, 9], [11]]
v = Variable(name='objects',
value=value,
fill_value=4,
dtype=ObjectType(int),
dimensions='values')
v.write(path_desired)
else:
v, path_actual, path_desired = [None] * 3
path_actual = MPI_COMM.bcast(path_actual)
path_desired = MPI_COMM.bcast(path_desired)
dest_mpi = OcgDist()
dest_mpi.create_dimension('values', 3, dist=True)
dest_mpi.update_dimension_bounds()
scattered = variable_scatter(v, dest_mpi)
outvc = VariableCollection(variables=[scattered])
with vm.scoped_by_emptyable('write', outvc):
if not vm.is_null:
outvc.write(path_actual)
if MPI_RANK == 0:
self.assertNcEqual(path_actual, path_desired)
def test_update_dimension_bounds_single_simple_dimension(self):
ompi = OcgDist(size=2)
ompi.create_dimension('d1', 2, dist=True)
ompi.update_dimension_bounds(min_elements=2)
d1 = ompi.get_dimension('d1')
for rank in range(2):
actual = ompi.get_dimension(d1.name, rank=rank)
if rank == 0:
self.assertFalse(actual.is_empty)
self.assertEqual(actual.bounds_local, (0, 2))
self.assertEqual(id(d1), id(actual))
else:
self.assertTrue(actual.is_empty)
self.assertEqual(actual.bounds_local, (0, 0))
self.assertNotEqual(id(d1), id(actual))
def test_get_distributed_slice(self):
self.add_barrier = False
for d in [True, False]:
dist = OcgDist()
dim = dist.create_dimension('five', 5, dist=d, src_idx='auto')
dist.update_dimension_bounds()
if not dim.is_empty:
self.assertEqual(dim.bounds_global, (0, 5))
if dim.dist:
if MPI_RANK > 1:
self.assertTrue(dim.is_empty)
else:
self.assertFalse(dim.is_empty)
with vm.scoped_by_emptyable('dim slice', dim):
if not vm.is_null:
sub = dim.get_distributed_slice(slice(1, 3))
else:
sub = None
if dim.dist:
if not dim.is_empty:
self.assertIsNotNone(dim._src_idx)
else:
self.assertIsNone(sub)
if MPI_SIZE == 2:
desired_emptiness = {0: False, 1: True}[MPI_RANK]
desired_bounds_local = {0: (0, 2), 1: (0, 0)}[MPI_RANK]
self.assertEqual(sub.is_empty, desired_emptiness)
self.assertEqual(sub.bounds_local, desired_bounds_local)
if MPI_SIZE >= 5 and 0 < MPI_RANK > 2:
self.assertTrue(sub is None or sub.is_empty)
else:
self.assertEqual(len(dim), 5)
self.assertEqual(dim.bounds_global, (0, 5))
self.assertEqual(dim.bounds_local, (0, 5))
dist = OcgDist()
dim = dist.create_dimension('five', 5, dist=True, src_idx='auto')
dist.update_dimension_bounds()
with vm.scoped_by_emptyable('five slice', dim):
if not vm.is_null:
sub2 = dim.get_distributed_slice(slice(2, 4))
else:
sub2 = None
if MPI_SIZE == 3 and MPI_RANK == 2:
self.assertIsNone(sub2)
def test_redistribute_by_src_idx(self):
if vm.size != 4:
raise SkipTest('vm.size != 4')
dist = OcgDist()
dim1 = dist.create_dimension('dim1', 5 * vm.size, dist=True)
dim2 = dist.create_dimension('dim2', 2, dist=False)
dist.update_dimension_bounds()
rank_value = np.arange(5) + (10 * (vm.rank + 1))
var1 = Variable(name='dvar1', value=rank_value, dimensions=dim1)
var2 = Variable(name='dvar2', dimensions=[dim1, dim2])
var1.parent.add_variable(var2)
path = self.get_temporary_file_path('out.nc')
var1.parent.write(path)
desired_idx = np.array([1, 7, 9, 10, 14])
vdesired_value = variable_gather(var1)
if vm.rank == 0:
desired_value = vdesired_value.get_value()[desired_idx]
desired_idx_ranks = {0: slice(1, 2),
1: [2, 4],
2: [0, 4]}
rd = RequestDataset(path)
rd.metadata['dimensions'][dim1.name]['dist'] = True
field = rd.create_field()
indvar = field[var1.name]
field[var2.name].load()
try:
rank_slice = desired_idx_ranks[vm.rank]
except KeyError:
sub = Variable(is_empty=True)
else:
sub = indvar[rank_slice]
self.barrier_print(sub.is_empty)
redistribute_by_src_idx(indvar, dim1.name, sub.dimensions_dict.get(dim1.name))
with vm.scoped_by_emptyable('gather for test', indvar):
if vm.is_null:
self.assertIn(vm.rank_global, [2, 3])
else:
self.assertIn(vm.rank_global, [0, 1])
for v in [indvar, indvar.parent[var2.name]]:
self.assertIsNone(v._value)
self.assertIsNone(v._mask)
self.assertIsNone(v._is_empty)
self.assertFalse(v._has_initialized_value)
self.rank_print(indvar)
actual_value = variable_gather(indvar)
if vm.rank == 0:
actual_value = actual_value.get_value()
self.assertNumpyAll(actual_value, desired_value)
def get_dist(self):
"""
:rtype: :class:`ocgis.OcgVM`
"""
ompi = OcgDist(size=vm.size, ranks=vm.ranks)
# ompi = OcgDist()
# Convert metadata into a grouping consistent with the MPI dimensions.
metadata = {None: self.metadata_source}
for group_index in iter_all_group_keys(metadata):
group_meta = get_group(metadata, group_index)
# Add the dimensions to the distribution object.
dimensions = self._get_dimensions_main_(group_meta)
for dimension_name, dimension_meta in list(group_meta['dimensions'].items()):
target_dimension = find_dimension_in_sequence(dimension_name, dimensions)
target_dimension.dist = group_meta['dimensions'][dimension_name].get('dist', False)
ompi.add_dimension(target_dimension, group=group_index)
try:
dimension_map = self.rd.dimension_map.get_group(group_index)
except DimensionMapError:
# Likely a user-provided dimension map.
continue
# dimension_map = get_group(self.rd.dimension_map, group_index, has_root=False)
distributed_dimension_name = self.get_distributed_dimension_name(dimension_map,
group_meta['dimensions'])
# Allow no distributed dimensions to be returned.
if distributed_dimension_name is not None:
for target_rank in range(ompi.size):
distributed_dimension = ompi.get_dimension(distributed_dimension_name, group=group_index,
rank=target_rank)
distributed_dimension.dist = True
# Add the variables to the distribution object.
for variable_name, variable_meta in list(group_meta['variables'].items()):
ompi.add_variable(variable_name, dimensions=variable_meta['dimensions'], group=group_index)
# tdk: this will have to be moved to account for slicing
ompi.update_dimension_bounds()
return ompi
def test_update_dimension_bounds_with_source_indexing(self):
dist_size = 5
dist = OcgDist(size=dist_size)
dist.create_dimension('dim', 11, dist=True, src_idx='auto')
dist.update_dimension_bounds()
actual = []
for rank in range(dist_size):
rank_dim = dist.get_dimension('dim', rank=rank)
actual.append(rank_dim._src_idx)
desired = [(0, 3), (3, 5), (5, 7), (7, 9), (9, 11)]
self.assertEqual(actual, desired)
def test_get_wrapped_state(self):
if sys.version_info.major == 3 and sys.version_info.minor == 5:
raise SkipTest('undefined behavior with Python 3.5')
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=True)
ompi.create_dimension('y', 1)
ompi.update_dimension_bounds()
values = [{'value': [-179, -90, 0, 90, 180], 'desired': WrappedState.WRAPPED},
{'value': [0, 90, 180, 270, 360], 'desired': WrappedState.UNWRAPPED},
{'value': [1, 2, 3, 4, 5], 'desired': WrappedState.UNKNOWN}]
kwds = {'values': values, 'crs': [Spherical(), None]}
for k in self.iter_product_keywords(kwds):
ompi = deepcopy(ompi)
if MPI_RANK == 0:
vx = Variable(name='x', value=k.values['value'], dimensions='x')
vy = Variable(name='y', value=[0], dimensions='y')
else:
vx, vy = [None] * 2
vx = variable_scatter(vx, ompi)
vy = variable_scatter(vy, ompi)
grid = Grid(vx, vy)
field = Field(grid=grid, crs=k.crs)
with vm.scoped_by_emptyable('wrap', field):
if not vm.is_null:
wrapped_state = field.wrapped_state
else:
wrapped_state = None
if not field.is_empty:
if k.crs is None:
self.assertIsNone(wrapped_state)
else:
self.assertIsNotNone(wrapped_state)
if k.crs is None or field.is_empty:
self.assertIsNone(wrapped_state)
else:
self.assertEqual(wrapped_state, k.values['desired'])
# Test with masked geometries.
values = [Point(350, 2), Point(-90, 5), Point(340, 5)]
mask = [True, False, True]
gvar = GeometryVariable(name='geom', value=values, mask=mask, dimensions='ngeom')
crs = Spherical()
wrapped_state = crs.get_wrapped_state(gvar)
self.assertEqual(wrapped_state, WrappedState.WRAPPED)
def test_get_distributed_slice_on_rank_subset(self):
"""Test with some a priori empty dimensions."""
if MPI_SIZE != 4:
raise SkipTest('MPI_SIZE != 4')
ompi = OcgDist()
dim = ompi.create_dimension('eight', 8, dist=True)
ompi.update_dimension_bounds()
sub = dim.get_distributed_slice(slice(2, 6))
live_ranks = get_nonempty_ranks(sub, vm)
if MPI_RANK in [1, 2]:
self.assertFalse(sub.is_empty)
else:
self.assertTrue(sub.is_empty)
self.assertEqual(sub.bounds_local, (0, 0))
self.assertEqual(sub.bounds_global, (0, 0))
with vm.scoped('live rank dim subset', live_ranks):
if not vm.is_null:
sub2 = sub.get_distributed_slice(slice(2, 4))
else:
sub2 = None
if MPI_RANK == 2:
self.assertEqual(sub2.bounds_local, (0, 2))
self.assertEqual(sub2.bounds_global, (0, 2))
self.assertFalse(sub2.is_empty)
else:
self.assertTrue(sub2 is None or sub2.is_empty)
# Try again w/out scoping.
if sub.is_empty:
with self.assertRaises(EmptyObjectError):
sub.get_distributed_slice(slice(2, 4))
def test_get_distributed_slice_on_rank_subset(self):
"""Test with some a priori empty dimensions."""
if MPI_SIZE != 4:
raise SkipTest('MPI_SIZE != 4')
ompi = OcgDist()
dim = ompi.create_dimension('eight', 8, dist=True)
ompi.update_dimension_bounds()
sub = dim.get_distributed_slice(slice(2, 6))
live_ranks = get_nonempty_ranks(sub, vm)
if MPI_RANK in [1, 2]:
self.assertFalse(sub.is_empty)
else:
self.assertTrue(sub.is_empty)
self.assertEqual(sub.bounds_local, (0, 0))
self.assertEqual(sub.bounds_global, (0, 0))
with vm.scoped('live rank dim subset', live_ranks):
if not vm.is_null:
sub2 = sub.get_distributed_slice(slice(2, 4))
else:
sub2 = None
if MPI_RANK == 2:
self.assertEqual(sub2.bounds_local, (0, 2))
self.assertEqual(sub2.bounds_global, (0, 2))
self.assertFalse(sub2.is_empty)
else:
self.assertTrue(sub2 is None or sub2.is_empty)
# Try again w/out scoping.
if sub.is_empty:
with self.assertRaises(EmptyObjectError):
sub.get_distributed_slice(slice(2, 4))
def create_slices_for_dimension(size, splits):
ompi = OcgDist(size=splits)
dimname = 'foo'
ompi.create_dimension(dimname, size, dist=True)
ompi.update_dimension_bounds()
slices = []
for rank in range(splits):
dimension = ompi.get_dimension(dimname, rank=rank)
slices.append(dimension.bounds_local)
return slices
def test_system_grid_chunking(self):
if vm.size != 4:
raise SkipTest('vm.size != 4')
from ocgis.spatial.grid_chunker import GridChunker
path = self.path_esmf_unstruct
rd_dst = RequestDataset(uri=path,
driver=DriverESMFUnstruct,
crs=Spherical(),
grid_abstraction='point',
grid_is_isomorphic=True)
rd_src = deepcopy(rd_dst)
resolution = 0.28125
chunk_wd = os.path.join(self.current_dir_output, 'chunks')
if vm.rank == 0:
os.mkdir(chunk_wd)
vm.barrier()
paths = {'wd': chunk_wd}
gc = GridChunker(rd_src,
rd_dst,
nchunks_dst=[8],
src_grid_resolution=resolution,
dst_grid_resolution=resolution,
optimized_bbox_subset=True,
paths=paths,
genweights=True)
gc.write_chunks()
dist = OcgDist()
local_ctr = Dimension(name='ctr', size=8, dist=True)
dist.add_dimension(local_ctr)
dist.update_dimension_bounds()
for ctr in range(local_ctr.bounds_local[0], local_ctr.bounds_local[1]):
ctr += 1
s = os.path.join(chunk_wd, 'split_src_{}.nc'.format(ctr))
d = os.path.join(chunk_wd, 'split_dst_{}.nc'.format(ctr))
sf = Field.read(s, driver=DriverESMFUnstruct)
df = Field.read(d, driver=DriverESMFUnstruct)
self.assertGreater(sf.grid.shape[0], df.grid.shape[0])
wgt = os.path.join(chunk_wd, 'esmf_weights_{}.nc'.format(ctr))
f = Field.read(wgt)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
with vm.scoped('merge weights', [0]):
if not vm.is_null:
merged_weights = self.get_temporary_file_path(
'merged_weights.nc')
gc.create_merged_weight_file(merged_weights, strict=False)
f = Field.read(merged_weights)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
def test_get_mask_from_intersects(self):
poly = wkt.loads(
'POLYGON((-98.26574367088608142 40.19952531645570559,-98.71764240506330168 39.54825949367089066,-99.26257911392406186 39.16281645569620906,-99.43536392405064817 38.64446202531645724,-98.78409810126584034 38.33876582278481493,-98.23916139240508016 37.71408227848101546,-97.77397151898735217 37.67420886075949937,-97.62776898734178133 38.15268987341772799,-98.39865506329114453 38.52484177215190186,-98.23916139240508016 39.33560126582278826,-97.73409810126582897 39.58813291139241386,-97.52143987341773368 40.27927215189873777,-97.52143987341773368 40.27927215189873777,-98.26574367088608142 40.19952531645570559))')
desired_mask = np.array([[True, True, False, True],
[True, False, True, True],
[True, True, False, True]])
dist = OcgDist()
xdim = dist.create_dimension('x', 4, dist=True)
ydim = dist.create_dimension('y', 3)
dist.create_dimension('bounds', 2)
dist.update_dimension_bounds()
if MPI_RANK == 0:
x = self.get_variable_x()
y = self.get_variable_y()
grid = Grid(x=x, y=y, abstraction='point', crs=WGS84())
pa = get_geometry_variable(grid)
else:
pa = None
pa = variable_scatter(pa, dist)
vm.create_subcomm_by_emptyable('test_get_mask_from_intersects', pa, is_current=True)
if vm.is_null:
self.assertTrue(pa.is_empty)
return
usi = [False]
if env.USE_SPATIAL_INDEX:
usi.append(True)
keywords = dict(use_spatial_index=usi)
for k in self.iter_product_keywords(keywords):
ret = pa.get_mask_from_intersects(poly, use_spatial_index=k.use_spatial_index)
desired_mask_local = desired_mask[slice(*ydim.bounds_local), slice(*xdim.bounds_local)]
if MPI_RANK > 1:
self.assertIsNone(ret)
else:
self.assertNumpyAll(desired_mask_local, ret)
# This does not test a parallel operation.
if MPI_RANK == 0:
# Test pre-masked values in geometry are okay for intersects operation.
value = [Point(1, 1), Point(2, 2), Point(3, 3)]
value = np.ma.array(value, mask=[False, True, False], dtype=object)
pa2 = GeometryVariable(value=value, dimensions='ngeom')
b = box(0, 0, 5, 5)
res = pa2.get_mask_from_intersects(b, use_spatial_index=k.use_spatial_index)
self.assertNumpyAll(res, value.mask)
def test_reduce_global(self):
pt = self.fixture(cindex=self.fixture_cindex(1), start_index=1)
self.assertEqual(pt.start_index, 1)
dist = OcgDist()
for d in pt.parent.dimensions.values():
d = d.copy()
if d.name == self.fixture_element_dimension.name:
d.dist = True
dist.add_dimension(d)
dist.update_dimension_bounds()
new_parent = variable_collection_scatter(pt.parent, dist)
vm.create_subcomm_by_emptyable('coordinate reduction',
new_parent,
is_current=True)
if vm.is_null:
return
pt.parent = new_parent
sub = pt.get_distributed_slice(slice(2, 5))
vm.create_subcomm_by_emptyable('distributed slice',
sub,
is_current=True)
if vm.is_null:
return
actual = sub.reduce_global()
actual_cindex = actual.cindex.extract()
actual_cindex = variable_gather(actual_cindex)
if vm.rank == 0:
actual_cindex = actual_cindex.get_value().flatten().tolist()
self.assertEqual(actual_cindex, [1, 2, 3])
gathered = [
variable_gather(c.extract()) for c in actual.coordinate_variables
]
if vm.rank == 0:
actual_coords = []
for c in gathered:
actual_coords.append(c.get_value().tolist())
desired = [[2.0, 3.0, 4.0], [8.0, 9.0, 10.0], [14.0, 15.0, 16.0]]
self.assertEqual(actual_coords, desired)
path = self.get_temporary_file_path('foo.nc')
actual.parent.write(path)
actual = Field.read(path)
self.assertEqual(actual['cindex'].attrs['start_index'], 1)
def create_dist(self, metadata=None):
"""
Create a distribution from global metadata. In general, this should not be overloaded by subclasses.
:param dict metadata: Global metadata to use for creating a distribution.
:rtype: :class:`ocgis.OcgDist`
"""
ompi = OcgDist(size=vm.size, ranks=vm.ranks)
# Convert metadata into a grouping consistent with the MPI dimensions.
if metadata is None:
metadata = self.metadata_source
metadata = {None: metadata}
for group_index in iter_all_group_keys(metadata):
group_meta = get_group(metadata, group_index)
# Add the dimensions to the distribution object.
dimensions = self.create_dimensions(group_meta)
# Only build a distribution if the group has more than one dimension.
if len(dimensions) == 0:
_ = ompi.get_group(group=group_index)
else:
for dimension_name, dimension_meta in list(group_meta['dimensions'].items()):
target_dimension = dimensions[dimension_name]
target_dimension.dist = group_meta['dimensions'][dimension_name].get('dist', False)
ompi.add_dimension(target_dimension, group=group_index)
try:
dimension_map = self.rd.dimension_map.get_group(group_index)
except DimensionMapError:
# Likely a user-provided dimension map.
continue
# dimension_map = get_group(self.rd.dimension_map, group_index, has_root=False)
distributed_dimension_name = self.get_distributed_dimension_name(dimension_map,
group_meta['dimensions'],
decomp_type=self.rd.decomp_type)
# Allow no distributed dimensions to be returned.
if distributed_dimension_name is not None:
for target_rank in range(ompi.size):
distributed_dimension = ompi.get_dimension(distributed_dimension_name, group=group_index,
rank=target_rank)
distributed_dimension.dist = True
ompi.update_dimension_bounds()
return ompi
def test_reduce_reindex_coordinate_variables(self):
self.add_barrier = False
dist = OcgDist()
dist.create_dimension('dim', 12, dist=True)
dist.update_dimension_bounds()
global_cindex_arr = np.array([4, 2, 1, 2, 1, 4, 1, 4, 2, 5, 6, 7])
if vm.rank == 0:
var_cindex = Variable('cindex',
value=global_cindex_arr,
dimensions='dim')
else:
var_cindex = None
var_cindex = variable_scatter(var_cindex, dist)
vm.create_subcomm_by_emptyable('test', var_cindex, is_current=True)
if vm.is_null:
return
raise_if_empty(var_cindex)
coords = np.array([
0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 100, 110, 120, 130, 140, 150
])
coords = Variable(name='coords', value=coords, dimensions='coord_dim')
new_cindex, u_indices = reduce_reindex_coordinate_variables(var_cindex)
desired = coords[global_cindex_arr].get_value()
if len(u_indices) > 0:
new_coords = coords[u_indices].get_value()
else:
new_coords = np.array([])
gathered_new_coords = vm.gather(new_coords)
gathered_new_cindex = vm.gather(new_cindex)
if vm.rank == 0:
gathered_new_coords = hgather(gathered_new_coords)
gathered_new_cindex = hgather(gathered_new_cindex)
actual = gathered_new_coords[gathered_new_cindex]
self.assertAsSetEqual(gathered_new_cindex.tolist(),
[2, 1, 0, 3, 4, 5])
desired_new_coords = [11, 22, 44, 55, 66, 77]
self.assertAsSetEqual(gathered_new_coords.tolist(),
desired_new_coords)
self.assertEqual(len(gathered_new_coords), len(desired_new_coords))
self.assertNumpyAll(actual, desired)
def test_create_ugid_global(self):
ompi = OcgDist()
m = ompi.create_dimension('m', 4)
n = ompi.create_dimension('n', 70, dist=True)
ompi.update_dimension_bounds()
gvar = GeometryVariable(name='geom', dimensions=(m, n))
ugid = gvar.create_ugid_global('gid')
if not gvar.is_empty:
self.assertEqual(gvar.dist, ugid.dist)
gathered = variable_gather(ugid)
if MPI_RANK == 0:
actual = gathered.get_value()
self.assertEqual(actual.size, len(set(actual.flatten().tolist())))
def test_get_unioned_spatial_average_parallel(self):
if MPI_SIZE != 8:
raise SkipTest('MPI_SIZE != 8')
dist = OcgDist()
geom_count = dist.create_dimension('geom_count', size=8, dist=True)
time_count = dist.create_dimension('time', size=3)
dist.update_dimension_bounds()
if not geom_count.is_empty:
gvar = GeometryVariable(
value=[Point(1.0, 1.0).buffer(MPI_RANK + 2)] * len(geom_count),
dimensions=geom_count)
value = np.zeros((len(time_count), len(geom_count)), dtype=float)
for ii in range(value.shape[0]):
value[ii] = [MPI_RANK + 1 + ii + 1] * len(geom_count)
data = Variable(name='data',
value=value,
dtype=float,
dimensions=[time_count, geom_count])
else:
gvar = GeometryVariable(dimensions=geom_count)
data = Variable(name='data',
dimensions=[time_count, geom_count],
dtype=float)
gvar.parent.add_variable(data)
self.assertTrue(gvar.is_empty == data.is_empty == gvar.parent.is_empty)
with vm.scoped_by_emptyable('union', gvar):
if vm.is_null:
unioned = None
else:
unioned = gvar.get_unioned(spatial_average='data')
if unioned is not None:
self.assertIsInstance(unioned, GeometryVariable)
actual = unioned.parent[data.name]
self.assertAlmostEqual(actual.get_value().max(),
5.5466666666666677)
else:
self.assertIsNone(unioned)
def test_variable_gather_bounded_source_index(self):
self.add_barrier = False
dist = OcgDist()
dist.create_dimension('long', 7, dist=True, src_idx='auto')
dist.create_dimension('short', 3, src_idx='auto')
dist.update_dimension_bounds()
if vm.rank == 0:
var = Variable(name='test', value=np.arange(21).reshape(7, 3), dimensions=['long', 'short'])
else:
var = None
svar = variable_scatter(var, dist)
with vm.scoped_by_emptyable('gather test', svar):
if vm.is_null:
return
gvar = variable_gather(svar)
if vm.rank == 0:
actual = gvar.dimensions[0]._src_idx
desired = (0, 7)
self.assertEqual(actual, desired)
else:
self.assertIsNone(gvar)
def test_get_intersects_one_rank_with_mask(self):
"""Test mask is created if one rank has a spatial mask."""
if MPI_SIZE != 2:
raise SkipTest('MPI_SIZE != 2')
if MPI_RANK == 0:
value = [1, 2]
else:
value = [3, 4]
ompi = OcgDist()
xdim = ompi.create_dimension('x', 4, dist=True)
ydim = ompi.create_dimension('y', 5, dist=False)
ompi.update_dimension_bounds()
x = Variable('x', value=value, dimensions=xdim)
y = Variable('y', value=[1, 2, 3, 4, 5], dimensions=ydim)
grid = Grid(x, y)
wkt_geom = 'Polygon ((0.72993630573248502 5.22484076433120936, 0.70318471337579691 0.67707006369426814, 2.70063694267515952 0.69490445859872629, 2.59363057324840796 2.54076433121019107, 4.52866242038216527 2.51401273885350296, 4.40382165605095466 5.34968152866241908, 0.72993630573248502 5.22484076433120936))'
subset_geom = wkt.loads(wkt_geom)
sub = grid.get_intersects(subset_geom)
path = self.get_temporary_file_path('foo.nc')
field = Field(grid=sub)
field.write(path)
with vm.scoped('mask count', [0]):
if not vm.is_null:
rd = RequestDataset(path)
out_field = rd.get()
target = out_field[out_field.grid._mask_name].get_value()
select = target != 0
self.assertEqual(select.sum(), 4)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time',
value=[1, 2, 3],
dtype=float,
dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x',
value=[9, 10, 11, 12],
dimensions='x',
dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y',
value=[13, 14, 15, 16, 17, 18, 19],
dimensions='y',
dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data',
value=np.random.rand(3, 2, 7, 4),
dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {
'x': {
'variable': 'x',
'bounds': 'x_bounds'
},
'y': {
'variable': 'y',
'bounds': 'y_bounds'
},
'time': {
'variable': 'time',
'bounds': 'the_time_bounds'
}
}
vc = Field(variables=[t, extra, x, y, data],
dimension_map=dimension_map,
is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel CSV write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.csv')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
vc = VariableCollection(variables=[t, extra, x, y, data])
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, iter_kwargs={'variable': 'data', 'followers': ['time', 'extra', 'y', 'x']},
driver=DriverCSV)
if MPI_RANK == 0:
desired = 169
with open(path, 'r') as f:
lines = f.readlines()
self.assertEqual(len(lines), desired)
def test_update_dimension_bounds(self):
ompi = OcgDist()
dim1 = ompi.create_dimension('five', 5, dist=True)
ompi.update_dimension_bounds()
if dim1.is_empty:
desired = (0, 0)
else:
desired = (0, 5)
self.assertEqual(dim1.bounds_global, desired)
if MPI_SIZE > 1:
if MPI_SIZE == 2:
if MPI_RANK == 0:
self.assertEqual(dim1.bounds_local, (0, 3))
else:
self.assertEqual(dim1.bounds_local, (3, 5))
# Test updating on single processor.
if MPI_SIZE == 1:
ompi = OcgDist(size=2)
ompi.create_dimension('five', 5, dist=True)
ompi.update_dimension_bounds()
dim = ompi.get_dimension('five')
self.assertEqual(dim.bounds_global, (0, 5))
for rank in range(2):
actual = ompi.get_dimension('five', rank=rank)
self.assertEqual(actual.bounds_global, (0, 5))
if rank == 0:
self.assertEqual(actual.bounds_local, (0, 3))
else:
self.assertEqual(actual.bounds_local, (3, 5))
# Test two dimensions.
ompi = OcgDist(size=2)
ompi.create_dimension('lat', 64, dist=True)
ompi.create_dimension('lon', 128, dist=True)
ompi.update_dimension_bounds()
for rank in range(2):
lat = ompi.get_dimension('lat', rank=rank)
self.assertEqual(lat.bounds_local, (0, 64))
lon = ompi.get_dimension('lon', rank=rank)
if rank == 0:
self.assertEqual(lon.bounds_local, (0, 64))
else:
self.assertEqual(lon.bounds_local, (64, 128))
def test_get_empty_ranks(self):
ompi = OcgDist(size=5)
ompi.create_dimension('four', 4, dist=True)
ompi.update_dimension_bounds()
self.assertEqual(ompi.get_empty_ranks(), (2, 3, 4))
self.assertEqual(ompi.get_empty_ranks(inverse=True), (0, 1))
