def test(self):
gs = self.get_grid_splitter()
desired_dst_grid_sum = gs.dst_grid.parent['data'].get_value().sum()
desired_dst_grid_sum = MPI_COMM.gather(desired_dst_grid_sum)
if MPI_RANK == 0:
desired_sum = np.sum(desired_dst_grid_sum)
desired = [{'y': slice(0, 180, None), 'x': slice(0, 240, None)},
{'y': slice(0, 180, None), 'x': slice(240, 480, None)},
{'y': slice(0, 180, None), 'x': slice(480, 720, None)},
{'y': slice(180, 360, None), 'x': slice(0, 240, None)},
{'y': slice(180, 360, None), 'x': slice(240, 480, None)},
{'y': slice(180, 360, None), 'x': slice(480, 720, None)}]
actual = list(gs.iter_dst_grid_slices())
self.assertEqual(actual, desired)
gs.write_subsets()
if MPI_RANK == 0:
rank_sums = []
for ctr in range(1, gs.nsplits_dst[0] * gs.nsplits_dst[1] + 1):
src_path = gs.create_full_path_from_template('src_template', index=ctr)
dst_path = gs.create_full_path_from_template('dst_template', index=ctr)
src_field = RequestDataset(src_path).get()
dst_field = RequestDataset(dst_path).get()
src_envelope_global = box(*src_field.grid.extent_global)
dst_envelope_global = box(*dst_field.grid.extent_global)
self.assertTrue(does_contain(src_envelope_global, dst_envelope_global))
actual = get_variable_names(src_field.data_variables)
self.assertIn('data', actual)
actual = get_variable_names(dst_field.data_variables)
self.assertIn('data', actual)
actual_data_sum = dst_field['data'].get_value().sum()
actual_data_sum = MPI_COMM.gather(actual_data_sum)
if MPI_RANK == 0:
actual_data_sum = np.sum(actual_data_sum)
rank_sums.append(actual_data_sum)
if MPI_RANK == 0:
self.assertAlmostEqual(desired_sum, np.sum(rank_sums))
index_path = gs.create_full_path_from_template('index_file')
self.assertTrue(os.path.exists(index_path))
MPI_COMM.Barrier()
index_path = gs.create_full_path_from_template('index_file')
index_field = RequestDataset(index_path).get()
self.assertTrue(len(list(index_field.keys())) > 2)
def test(self):
gs = self.fixture_grid_chunker()
desired_dst_grid_sum = gs.dst_grid.parent['data'].get_value().sum()
desired_dst_grid_sum = MPI_COMM.gather(desired_dst_grid_sum)
if vm.rank == 0:
desired_sum = np.sum(desired_dst_grid_sum)
desired = [{'y': slice(0, 180, None), 'x': slice(0, 240, None)},
{'y': slice(0, 180, None), 'x': slice(240, 480, None)},
{'y': slice(0, 180, None), 'x': slice(480, 720, None)},
{'y': slice(180, 360, None), 'x': slice(0, 240, None)},
{'y': slice(180, 360, None), 'x': slice(240, 480, None)},
{'y': slice(180, 360, None), 'x': slice(480, 720, None)}]
actual = list(gs.iter_dst_grid_slices())
self.assertEqual(actual, desired)
gs.write_chunks()
if vm.rank == 0:
rank_sums = []
for ctr in range(1, gs.nchunks_dst[0] * gs.nchunks_dst[1] + 1):
src_path = gs.create_full_path_from_template('src_template', index=ctr)
dst_path = gs.create_full_path_from_template('dst_template', index=ctr)
src_field = RequestDataset(src_path).get()
dst_field = RequestDataset(dst_path).get()
src_envelope_global = box(*src_field.grid.extent_global)
dst_envelope_global = box(*dst_field.grid.extent_global)
self.assertTrue(does_contain(src_envelope_global, dst_envelope_global))
actual = get_variable_names(src_field.data_variables)
self.assertIn('data', actual)
actual = get_variable_names(dst_field.data_variables)
self.assertIn('data', actual)
actual_data_sum = dst_field['data'].get_value().sum()
actual_data_sum = MPI_COMM.gather(actual_data_sum)
if MPI_RANK == 0:
actual_data_sum = np.sum(actual_data_sum)
rank_sums.append(actual_data_sum)
if vm.rank == 0:
self.assertAlmostEqual(desired_sum, np.sum(rank_sums))
index_path = gs.create_full_path_from_template('index_file')
self.assertTrue(os.path.exists(index_path))
vm.barrier()
index_path = gs.create_full_path_from_template('index_file')
index_field = RequestDataset(index_path).get()
self.assertTrue(len(list(index_field.keys())) > 2)
def test_get_intersection_state_boundaries(self):
path_shp = self.path_state_boundaries
geoms = []
with fiona.open(path_shp) as source:
for record in source:
geom = shape(record['geometry'])
geoms.append(geom)
gvar = GeometryVariable(value=geoms, dimensions='ngeom')
gvar_sub = gvar.get_unioned()
if gvar_sub is not None:
subset = gvar_sub.get_value().flatten()[0]
else:
subset = None
subset = MPI_COMM.bcast(subset)
resolution = 2.0
keywords = dict(with_bounds=[False])
for k in self.iter_product_keywords(keywords):
grid = self.get_gridxy_global(resolution=resolution,
with_bounds=k.with_bounds)
res = grid.get_intersection(subset)
if not res.is_empty:
self.assertTrue(res.get_mask().any())
else:
self.assertIsInstance(res, GeometryVariable)
if k.with_bounds:
area = res.area
if area is None:
area = 0.0
else:
area = area.sum()
areas = MPI_COMM.gather(area)
if MPI_RANK == 0:
area_global = sum(areas)
self.assertAlmostEqual(area_global, 1096.0819224080542)
else:
mask = res.get_mask()
if mask is None:
masked = 0
else:
masked = mask.sum()
masked = MPI_COMM.gather(masked)
if MPI_RANK == 0:
total_masked = sum(masked)
self.assertEqual(total_masked, 858)
def test_system_with_distributed_dimensions_from_file_shapefile(self):
"""Test a distributed read from file."""
path = self.path_state_boundaries
# These are the desired values.
with vm.scoped('desired data write', [0]):
if not vm.is_null:
rd_desired = RequestDataset(uri=path, driver=DriverVector)
var_desired = SourcedVariable(name='STATE_NAME',
request_dataset=rd_desired)
value_desired = var_desired.get_value().tolist()
self.assertEqual(len(value_desired), 51)
rd = RequestDataset(uri=path, driver=DriverVector)
fvar = SourcedVariable(name='STATE_NAME', request_dataset=rd)
self.assertEqual(len(rd.driver.dist.get_group()['dimensions']), 1)
self.assertTrue(fvar.dimensions[0].dist)
self.assertIsNotNone(fvar.get_value())
if MPI_SIZE > 1:
self.assertLessEqual(fvar.shape[0], 26)
values = MPI_COMM.gather(fvar.get_value())
if MPI_RANK == 0:
values = hgather(values)
self.assertEqual(values.tolist(), value_desired)
else:
self.assertIsNone(values)
def test_system_with_distributed_dimensions_from_file_shapefile(self):
"""Test a distributed read from file."""
path = self.path_state_boundaries
# These are the desired values.
with vm.scoped('desired data write', [0]):
if not vm.is_null:
rd_desired = RequestDataset(uri=path, driver=DriverVector)
var_desired = SourcedVariable(name='STATE_NAME', request_dataset=rd_desired)
value_desired = var_desired.get_value().tolist()
self.assertEqual(len(value_desired), 51)
rd = RequestDataset(uri=path, driver=DriverVector)
fvar = SourcedVariable(name='STATE_NAME', request_dataset=rd)
self.assertEqual(len(rd.driver.dist.get_group()['dimensions']), 1)
self.assertTrue(fvar.dimensions[0].dist)
self.assertIsNotNone(fvar.get_value())
if MPI_SIZE > 1:
self.assertLessEqual(fvar.shape[0], 26)
values = MPI_COMM.gather(fvar.get_value())
if MPI_RANK == 0:
values = hgather(values)
self.assertEqual(values.tolist(), value_desired)
else:
self.assertIsNone(values)
def test_system_spatial_averaging_from_file(self):
rd_nc = self.test_data.get_rd('cancm4_tas')
rd_shp = RequestDataset(self.path_state_boundaries)
field_shp = rd_shp.get()
actual = field_shp.dimension_map.get_variable(DMK.GEOM)
self.assertIsNotNone(actual)
actual = field_shp.dimension_map.get_dimension(DMK.GEOM)
self.assertEqual(len(actual), 1)
self.assertEqual(field_shp.crs, WGS84())
try:
index_geom = np.where(
field_shp['STATE_NAME'].get_value() == 'Nebraska')[0][0]
except IndexError:
# Not found on rank.
polygon_field = None
else:
polygon_field = field_shp.get_field_slice({'geom': index_geom})
polygon_field = MPI_COMM.gather(polygon_field)
if MPI_RANK == 0:
for p in polygon_field:
if p is not None:
polygon_field = p
break
polygon_field = MPI_COMM.bcast(polygon_field)
polygon_field.unwrap()
polygon = polygon_field.geom.get_value()[0]
field_nc = rd_nc.get()
sub_field_nc = field_nc.get_field_slice({'time': slice(0, 10)})
self.assertEqual(sub_field_nc['tas']._dimensions,
field_nc['tas']._dimensions)
sub = sub_field_nc.grid.get_intersects(polygon)
# When split across two processes, there are floating point summing differences.
desired = {1: 2734.5195, 2: 2740.4014}
with vm.scoped_by_emptyable('grid intersects', sub):
if not vm.is_null:
abstraction_geometry = sub.get_abstraction_geometry()
sub.parent.add_variable(abstraction_geometry, force=True)
unioned = abstraction_geometry.get_unioned(
spatial_average='tas')
if unioned is not None:
tas = unioned.parent['tas']
self.assertFalse(tas.is_empty)
self.assertAlmostEqual(tas.get_value().sum(),
desired[vm.size],
places=4)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_system_spatial_averaging_from_file(self):
rd_nc = self.test_data.get_rd('cancm4_tas')
rd_shp = RequestDataset(self.path_state_boundaries)
field_shp = rd_shp.get()
actual = field_shp.dimension_map.get_variable(DMK.GEOM)
self.assertIsNotNone(actual)
actual = field_shp.dimension_map.get_dimension(DMK.GEOM)
self.assertEqual(len(actual), 1)
self.assertEqual(field_shp.crs, WGS84())
try:
index_geom = np.where(field_shp['STATE_NAME'].get_value() == 'Nebraska')[0][0]
except IndexError:
# Not found on rank.
polygon_field = None
else:
polygon_field = field_shp.get_field_slice({'geom': index_geom})
polygon_field = MPI_COMM.gather(polygon_field)
if MPI_RANK == 0:
for p in polygon_field:
if p is not None:
polygon_field = p
break
polygon_field = MPI_COMM.bcast(polygon_field)
polygon_field.unwrap()
polygon = polygon_field.geom.get_value()[0]
field_nc = rd_nc.get()
sub_field_nc = field_nc.get_field_slice({'time': slice(0, 10)})
self.assertEqual(sub_field_nc['tas']._dimensions, field_nc['tas']._dimensions)
sub = sub_field_nc.grid.get_intersects(polygon)
# When split across two processes, there are floating point summing differences.
desired = {1: 2734.5195, 2: 2740.4014}
with vm.scoped_by_emptyable('grid intersects', sub):
if not vm.is_null:
abstraction_geometry = sub.get_abstraction_geometry()
sub.parent.add_variable(abstraction_geometry, force=True)
unioned = abstraction_geometry.get_unioned(spatial_average='tas')
if unioned is not None:
tas = unioned.parent['tas']
self.assertFalse(tas.is_empty)
self.assertAlmostEqual(tas.get_value().sum(), desired[vm.size], places=4)