def test_reduce_reindex_coordinate_variables(self):
self.add_barrier = False
dist = OcgDist()
dist.create_dimension('dim', 12, dist=True)
dist.update_dimension_bounds()
global_cindex_arr = np.array([4, 2, 1, 2, 1, 4, 1, 4, 2, 5, 6, 7])
if vm.rank == 0:
var_cindex = Variable('cindex',
value=global_cindex_arr,
dimensions='dim')
else:
var_cindex = None
var_cindex = variable_scatter(var_cindex, dist)
vm.create_subcomm_by_emptyable('test', var_cindex, is_current=True)
if vm.is_null:
return
raise_if_empty(var_cindex)
coords = np.array([
0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 100, 110, 120, 130, 140, 150
])
coords = Variable(name='coords', value=coords, dimensions='coord_dim')
new_cindex, u_indices = reduce_reindex_coordinate_variables(var_cindex)
desired = coords[global_cindex_arr].get_value()
if len(u_indices) > 0:
new_coords = coords[u_indices].get_value()
else:
new_coords = np.array([])
gathered_new_coords = vm.gather(new_coords)
gathered_new_cindex = vm.gather(new_cindex)
if vm.rank == 0:
gathered_new_coords = hgather(gathered_new_coords)
gathered_new_cindex = hgather(gathered_new_cindex)
actual = gathered_new_coords[gathered_new_cindex]
self.assertAsSetEqual(gathered_new_cindex.tolist(),
[2, 1, 0, 3, 4, 5])
desired_new_coords = [11, 22, 44, 55, 66, 77]
self.assertAsSetEqual(gathered_new_coords.tolist(),
desired_new_coords)
self.assertEqual(len(gathered_new_coords), len(desired_new_coords))
self.assertNumpyAll(actual, desired)
def test_system_with_distributed_dimensions_from_file_shapefile(self):
"""Test a distributed read from file."""
path = self.path_state_boundaries
# These are the desired values.
with vm.scoped('desired data write', [0]):
if not vm.is_null:
rd_desired = RequestDataset(uri=path, driver=DriverVector)
var_desired = SourcedVariable(name='STATE_NAME',
request_dataset=rd_desired)
value_desired = var_desired.get_value().tolist()
self.assertEqual(len(value_desired), 51)
rd = RequestDataset(uri=path, driver=DriverVector)
fvar = SourcedVariable(name='STATE_NAME', request_dataset=rd)
self.assertEqual(len(rd.driver.dist.get_group()['dimensions']), 1)
self.assertTrue(fvar.dimensions[0].dist)
self.assertIsNotNone(fvar.get_value())
if MPI_SIZE > 1:
self.assertLessEqual(fvar.shape[0], 26)
values = MPI_COMM.gather(fvar.get_value())
if MPI_RANK == 0:
values = hgather(values)
self.assertEqual(values.tolist(), value_desired)
else:
self.assertIsNone(values)
def _gc_iter_dst_grid_slices_(grid_chunker):
# TODO: This method uses some global gathers which is not ideal.
# Destination splitting works off center coordinates only.
pgc = grid_chunker.dst_grid.abstractions_available['point']
# Use the unique center values to break the grid into pieces. This ensures that nearby grid cell are close
# spatially. If we just break the grid into pieces w/out using unique values, the points may be scattered which
# does not optimize the spatial coverage of the source grid.
center_lat = pgc.y.get_value()
# ucenter_lat = np.unique(center_lat)
ucenter_lat = create_unique_global_array(center_lat)
ucenter_lat = vm.gather(ucenter_lat)
if vm.rank == 0:
ucenter_lat = hgather(ucenter_lat)
ucenter_lat.sort()
ucenter_splits = np.array_split(ucenter_lat, grid_chunker.nchunks_dst[0])
else:
ucenter_splits = [None] * grid_chunker.nchunks_dst[0]
for ucenter_split in ucenter_splits:
ucenter_split = vm.bcast(ucenter_split)
select = np.zeros_like(center_lat, dtype=bool)
for v in ucenter_split.flat:
select = np.logical_or(select, center_lat == v)
yield select
def test_create_unique_global_array(self):
dist = OcgDist()
dist.create_dimension('dim', 9, dist=True)
dist.update_dimension_bounds()
values = [
[4, 2, 1, 2, 1, 4, 1, 4, 2],
[44, 25, 16, 27, 18, 49, 10, 41, 22],
[44, 25, 16, 27, 44, 49, 10, 41, 44],
[1, 1, 1, 1, 1, 1, 1, 1, 1]
]
for v in values:
if vm.rank == 0:
index = Variable(name='cindex', value=v, dimensions='dim')
desired = np.unique(index.get_value())
desired_length = len(desired)
else:
index = None
index = variable_scatter(index, dist)
with vm.scoped_by_emptyable('not empty', index):
if not vm.is_null:
uvar = create_unique_global_array(index.get_value())
uvar_gathered = vm.gather(uvar)
if vm.rank == 0:
uvar_gathered = hgather(uvar_gathered)
self.assertEqual(len(uvar_gathered), desired_length)
self.assertEqual(set(uvar_gathered), set(desired))
def test_create_unique_global_array(self):
dist = OcgDist()
dist.create_dimension('dim', 9, dist=True)
dist.update_dimension_bounds()
values = [[4, 2, 1, 2, 1, 4, 1, 4, 2],
[44, 25, 16, 27, 18, 49, 10, 41, 22],
[44, 25, 16, 27, 44, 49, 10, 41, 44],
[1, 1, 1, 1, 1, 1, 1, 1, 1]]
for v in values:
if vm.rank == 0:
index = Variable(name='cindex', value=v, dimensions='dim')
desired = np.unique(index.get_value())
desired_length = len(desired)
else:
index = None
index = variable_scatter(index, dist)
with vm.scoped_by_emptyable('not empty', index):
if not vm.is_null:
uvar = create_unique_global_array(index.get_value())
uvar_gathered = vm.gather(uvar)
if vm.rank == 0:
uvar_gathered = hgather(uvar_gathered)
self.assertEqual(len(uvar_gathered), desired_length)
self.assertEqual(set(uvar_gathered), set(desired))
def test_system_with_distributed_dimensions_from_file_shapefile(self):
"""Test a distributed read from file."""
path = self.path_state_boundaries
# These are the desired values.
with vm.scoped('desired data write', [0]):
if not vm.is_null:
rd_desired = RequestDataset(uri=path, driver=DriverVector)
var_desired = SourcedVariable(name='STATE_NAME', request_dataset=rd_desired)
value_desired = var_desired.get_value().tolist()
self.assertEqual(len(value_desired), 51)
rd = RequestDataset(uri=path, driver=DriverVector)
fvar = SourcedVariable(name='STATE_NAME', request_dataset=rd)
self.assertEqual(len(rd.driver.dist.get_group()['dimensions']), 1)
self.assertTrue(fvar.dimensions[0].dist)
self.assertIsNotNone(fvar.get_value())
if MPI_SIZE > 1:
self.assertLessEqual(fvar.shape[0], 26)
values = MPI_COMM.gather(fvar.get_value())
if MPI_RANK == 0:
values = hgather(values)
self.assertEqual(values.tolist(), value_desired)
else:
self.assertIsNone(values)
def test_reduce_reindex_coordinate_index(self):
dist = OcgDist()
dist.create_dimension('dim', 12, dist=True)
dist.update_dimension_bounds()
global_cindex_arr = np.array([4, 2, 1, 2, 1, 4, 1, 4, 2, 5, 6, 7])
if vm.rank == 0:
var_cindex = Variable('cindex', value=global_cindex_arr, dimensions='dim')
else:
var_cindex = None
var_cindex = variable_scatter(var_cindex, dist)
vm.create_subcomm_by_emptyable('test', var_cindex, is_current=True)
if vm.is_null:
return
raise_if_empty(var_cindex)
coords = np.array([0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 100, 110, 120, 130, 140, 150])
coords = Variable(name='coords', value=coords, dimensions='coord_dim')
new_cindex, u_indices = reduce_reindex_coordinate_index(var_cindex)
desired = coords[global_cindex_arr].get_value()
if len(u_indices) > 0:
new_coords = coords[u_indices].get_value()
else:
new_coords = np.array([])
gathered_new_coords = vm.gather(new_coords)
gathered_new_cindex = vm.gather(new_cindex)
if vm.rank == 0:
gathered_new_coords = hgather(gathered_new_coords)
gathered_new_cindex = hgather(gathered_new_cindex)
actual = gathered_new_coords[gathered_new_cindex]
self.assertAsSetEqual(gathered_new_cindex.tolist(), [2, 1, 0, 3, 4, 5])
desired_new_coords = [11, 22, 44, 55, 66, 77]
self.assertAsSetEqual(gathered_new_coords.tolist(), desired_new_coords)
self.assertEqual(len(gathered_new_coords), len(desired_new_coords))
self.assertNumpyAll(actual, desired)
def test_arange_from_dimension(self):
dist = OcgDist()
dim = dist.create_dimension('dim', size=7, dist=True)
dist.update_dimension_bounds()
actual = arange_from_dimension(dim, start=2, dtype=np.int64)
actual = vm.gather(actual)
if vm.rank == 0:
actual = hgather(actual)
desired = np.arange(2, 9, dtype=np.int64)
self.assertNumpyAll(actual, desired)
def test_arange_from_dimension(self):
dist = OcgDist()
dim = dist.create_dimension('dim', size=7, dist=True)
dist.update_dimension_bounds()
actual = arange_from_dimension(dim, start=2, dtype=np.int64)
actual = vm.gather(actual)
if vm.rank == 0:
actual = hgather(actual)
desired = np.arange(2, 9, dtype=np.int64)
self.assertNumpyAll(actual, desired)
def test_create_nd_slices2(self):
size = (1, 1)
shape = (4, 3)
actual = create_nd_slices(size, shape)
self.assertEqual(actual, ((slice(0, 4, None), slice(0, 3, None)),))
size = (2, 1)
shape = (4, 3)
actual = create_nd_slices(size, shape)
self.assertEqual(actual, ((slice(0, 2, None), slice(0, 3, None)), (slice(2, 4, None), slice(0, 3, None))))
size = (4, 2)
shape = (4, 3)
actual = create_nd_slices(size, shape)
to_test = np.arange(12).reshape(*shape)
pieces = []
for a in actual:
pieces.append(to_test[a].reshape(-1))
self.assertNumpyAll(hgather(pieces).reshape(*shape), to_test)
def test_system_spatial_averaging_through_operations_state_boundaries(self):
if MPI_SIZE != 8:
raise SkipTest('MPI_SIZE != 8')
ntime = 3
# Get the exact field value for the state's representative center.
with vm.scoped([0]):
if MPI_RANK == 0:
states = RequestDataset(self.path_state_boundaries, driver='vector').get()
states.update_crs(env.DEFAULT_COORDSYS)
fill = np.zeros((states.geom.shape[0], 2))
for idx, geom in enumerate(states.geom.get_value().flat):
centroid = geom.centroid
fill[idx, :] = centroid.x, centroid.y
exact_states = create_exact_field_value(fill[:, 0], fill[:, 1])
state_ugid = states['UGID'].get_value()
area = states.geom.area
keywords = {
'spatial_operation': [
'clip',
'intersects'
],
'aggregate': [
True,
False
],
'wrapped': [True, False],
'output_format': [
OutputFormatName.OCGIS,
'csv',
'csv-shp',
'shp'
],
}
# total_iterations = len(list(self.iter_product_keywords(keywords)))
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# barrier_print(k)
# if ctr % 1 == 0:
# if vm.is_root:
# print('Iteration {} of {}...'.format(ctr + 1, total_iterations))
with vm.scoped([0]):
if vm.is_root:
grid = create_gridxy_global(resolution=1.0, dist=False, wrapped=k.wrapped)
field = create_exact_field(grid, 'foo', ntime=ntime)
path = self.get_temporary_file_path('foo.nc')
field.write(path)
else:
path = None
path = MPI_COMM.bcast(path)
rd = RequestDataset(path)
ops = OcgOperations(dataset=rd, geom='state_boundaries', spatial_operation=k.spatial_operation,
aggregate=k.aggregate, output_format=k.output_format, prefix=str(ctr),
# geom_select_uid=[8]
)
ret = ops.execute()
# Test area is preserved for a problem element during union. The union's geometry was not fully represented
# in the output.
if k.output_format == 'shp' and k.aggregate and k.spatial_operation == 'clip':
with vm.scoped([0]):
if vm.is_root:
inn = RequestDataset(ret).get()
inn_ugid_idx = np.where(inn['UGID'].get_value() == 8)[0][0]
ugid_idx = np.where(state_ugid == 8)[0][0]
self.assertAlmostEqual(inn.geom.get_value()[inn_ugid_idx].area, area[ugid_idx], places=2)
# Test the overview geometry shapefile is written.
if k.output_format == 'shp':
directory = os.path.split(ret)[0]
contents = os.listdir(directory)
actual = ['_ugid.shp' in c for c in contents]
self.assertTrue(any(actual))
elif k.output_format == 'csv-shp':
directory = os.path.split(ret)[0]
directory = os.path.join(directory, 'shp')
contents = os.listdir(directory)
actual = ['_ugid.shp' in c for c in contents]
self.assertTrue(any(actual))
if not k.aggregate:
actual = ['_gid.shp' in c for c in contents]
self.assertTrue(any(actual))
if k.output_format == OutputFormatName.OCGIS:
geom_keys = ret.children.keys()
all_geom_keys = vm.gather(np.array(geom_keys))
if vm.is_root:
all_geom_keys = hgather(all_geom_keys)
self.assertEqual(len(np.unique(all_geom_keys)), 51)
if k.aggregate:
actual = Dict()
for field, container in ret.iter_fields(yield_container=True):
if not field.is_empty:
ugid = container.geom.ugid.get_value()[0]
actual[ugid]['actual'] = field.data_variables[0].get_value()
actual[ugid]['area'] = container.geom.area[0]
actual = vm.gather(actual)
if vm.is_root:
actual = dgather(actual)
ares = []
actual_areas = []
for ugid_key, v in actual.items():
ugid_idx = np.where(state_ugid == ugid_key)[0][0]
desired = exact_states[ugid_idx]
actual_areas.append(v['area'])
for tidx in range(ntime):
are = np.abs((desired + ((tidx + 1) * 10)) - v['actual'][tidx, 0])
ares.append(are)
if k.spatial_operation == 'clip':
diff = np.abs(np.array(area) - np.array(actual_areas))
self.assertLess(np.max(diff), 1e-6)
self.assertLess(np.mean(diff), 1e-6)
# Test relative errors.
self.assertLess(np.max(ares), 0.031)
self.assertLess(np.mean(ares), 0.009)
