def test_system_geometry_identifer_added(self):
"""Test geometry identifier is added for linked dataset geometry formats."""
field = self.get_field()
gid_name = HeaderName.ID_GEOMETRY
self.assertNotIn(gid_name, field)
ops = OcgOperations(dataset=field, output_format=constants.OutputFormatName.CSV_SHAPEFILE)
ret = ops.execute()
csv_field = RequestDataset(ret).get()
self.assertIn(gid_name, list(csv_field.keys()))
shp_path = os.path.join(ops.dir_output, ops.prefix, 'shp', ops.prefix + '_gid.shp')
shp_field = RequestDataset(shp_path).get()
self.assertIn(gid_name, list(shp_field.keys()))
def test_system_user_geometry_identifer_added(self):
"""Test geometry identifier is added for linked dataset geometry formats."""
field = self.get_field(crs=WGS84())
subset_geom = Point(field.grid.x.get_value()[0], field.grid.y.get_value()[0]).buffer(0.1)
ops = OcgOperations(dataset=field, geom=subset_geom, output_format=constants.OutputFormatName.CSV_SHAPEFILE)
ret = ops.execute()
ugid_name = HeaderName.ID_SELECTION_GEOMETRY
csv_field = RequestDataset(ret).get()
self.assertIn(ugid_name, list(csv_field.keys()))
shp_path = os.path.join(ops.dir_output, ops.prefix, 'shp', ops.prefix + '_gid.shp')
shp_field = RequestDataset(shp_path).get()
self.assertIn(ugid_name, list(shp_field.keys()))
shp_path = os.path.join(ops.dir_output, ops.prefix, 'shp', ops.prefix + '_ugid.shp')
shp_field = RequestDataset(shp_path).get()
self.assertIn(ugid_name, list(shp_field.keys()))
def test(self):
gs = self.fixture_grid_chunker()
desired_dst_grid_sum = gs.dst_grid.parent['data'].get_value().sum()
desired_dst_grid_sum = MPI_COMM.gather(desired_dst_grid_sum)
if vm.rank == 0:
desired_sum = np.sum(desired_dst_grid_sum)
desired = [{'y': slice(0, 180, None), 'x': slice(0, 240, None)},
{'y': slice(0, 180, None), 'x': slice(240, 480, None)},
{'y': slice(0, 180, None), 'x': slice(480, 720, None)},
{'y': slice(180, 360, None), 'x': slice(0, 240, None)},
{'y': slice(180, 360, None), 'x': slice(240, 480, None)},
{'y': slice(180, 360, None), 'x': slice(480, 720, None)}]
actual = list(gs.iter_dst_grid_slices())
self.assertEqual(actual, desired)
gs.write_chunks()
if vm.rank == 0:
rank_sums = []
for ctr in range(1, gs.nchunks_dst[0] * gs.nchunks_dst[1] + 1):
src_path = gs.create_full_path_from_template('src_template', index=ctr)
dst_path = gs.create_full_path_from_template('dst_template', index=ctr)
src_field = RequestDataset(src_path).get()
dst_field = RequestDataset(dst_path).get()
src_envelope_global = box(*src_field.grid.extent_global)
dst_envelope_global = box(*dst_field.grid.extent_global)
self.assertTrue(does_contain(src_envelope_global, dst_envelope_global))
actual = get_variable_names(src_field.data_variables)
self.assertIn('data', actual)
actual = get_variable_names(dst_field.data_variables)
self.assertIn('data', actual)
actual_data_sum = dst_field['data'].get_value().sum()
actual_data_sum = MPI_COMM.gather(actual_data_sum)
if MPI_RANK == 0:
actual_data_sum = np.sum(actual_data_sum)
rank_sums.append(actual_data_sum)
if vm.rank == 0:
self.assertAlmostEqual(desired_sum, np.sum(rank_sums))
index_path = gs.create_full_path_from_template('index_file')
self.assertTrue(os.path.exists(index_path))
vm.barrier()
index_path = gs.create_full_path_from_template('index_file')
index_field = RequestDataset(index_path).get()
self.assertTrue(len(list(index_field.keys())) > 2)
def test(self):
gs = self.get_grid_splitter()
desired_dst_grid_sum = gs.dst_grid.parent['data'].get_value().sum()
desired_dst_grid_sum = MPI_COMM.gather(desired_dst_grid_sum)
if MPI_RANK == 0:
desired_sum = np.sum(desired_dst_grid_sum)
desired = [{'y': slice(0, 180, None), 'x': slice(0, 240, None)},
{'y': slice(0, 180, None), 'x': slice(240, 480, None)},
{'y': slice(0, 180, None), 'x': slice(480, 720, None)},
{'y': slice(180, 360, None), 'x': slice(0, 240, None)},
{'y': slice(180, 360, None), 'x': slice(240, 480, None)},
{'y': slice(180, 360, None), 'x': slice(480, 720, None)}]
actual = list(gs.iter_dst_grid_slices())
self.assertEqual(actual, desired)
gs.write_subsets()
if MPI_RANK == 0:
rank_sums = []
for ctr in range(1, gs.nsplits_dst[0] * gs.nsplits_dst[1] + 1):
src_path = gs.create_full_path_from_template('src_template', index=ctr)
dst_path = gs.create_full_path_from_template('dst_template', index=ctr)
src_field = RequestDataset(src_path).get()
dst_field = RequestDataset(dst_path).get()
src_envelope_global = box(*src_field.grid.extent_global)
dst_envelope_global = box(*dst_field.grid.extent_global)
self.assertTrue(does_contain(src_envelope_global, dst_envelope_global))
actual = get_variable_names(src_field.data_variables)
self.assertIn('data', actual)
actual = get_variable_names(dst_field.data_variables)
self.assertIn('data', actual)
actual_data_sum = dst_field['data'].get_value().sum()
actual_data_sum = MPI_COMM.gather(actual_data_sum)
if MPI_RANK == 0:
actual_data_sum = np.sum(actual_data_sum)
rank_sums.append(actual_data_sum)
if MPI_RANK == 0:
self.assertAlmostEqual(desired_sum, np.sum(rank_sums))
index_path = gs.create_full_path_from_template('index_file')
self.assertTrue(os.path.exists(index_path))
MPI_COMM.Barrier()
index_path = gs.create_full_path_from_template('index_file')
index_field = RequestDataset(index_path).get()
self.assertTrue(len(list(index_field.keys())) > 2)
def test_write_variable_collection(self):
# Attempt to write without a geometry variable.
v = Variable('a', value=[1, 2], dimensions='bb')
field = Field(variables=v)
path = self.get_temporary_file_path('out.shp')
with self.assertRaises(ValueError):
field.write(path, driver=DriverVector)
# Test writing a field with two-dimensional geometry storage.
value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)]
gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms')
gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)])
var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a'])
var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat'])
var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon'])
var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat'])
field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data'])
path = self.get_temporary_file_path('2d.shp')
field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertTrue(len(read) > 2)
self.assertEqual(list(read.keys()),
['data', 'some_lats', 'some_lons', constants.DimensionName.GEOMETRY_DIMENSION])
# Test writing a subset of the variables.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1])
field.write(path, variable_names=['keep'], driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertNotIn('remove', read)
# Test using append.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2])
for idx in range(3):
sub = field[{'points': idx}]
if idx == 0:
write_mode = MPIWriteMode.WRITE
else:
write_mode = MPIWriteMode.APPEND
sub.write(path, write_mode=write_mode, driver=DriverVector)
self.assertOGRFileLength(path, idx + 1)
def test_write_variable_collection(self):
# Attempt to write without a geometry variable.
v = Variable('a', value=[1, 2], dimensions='bb')
field = Field(variables=v)
path = self.get_temporary_file_path('out.shp')
with self.assertRaises(ValueError):
field.write(path, driver=DriverVector)
# Test writing a field with two-dimensional geometry storage.
value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)]
gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms')
gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)])
var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a'])
var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat'])
var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon'])
var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat'])
field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data'])
path = self.get_temporary_file_path('2d.shp')
field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertTrue(len(read) > 2)
self.assertEqual(list(read.keys()),
['data', 'some_lats', 'some_lons', constants.VariableName.GEOMETRY_VARIABLE])
# Test writing a subset of the variables.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1])
field.write(path, variable_names=['keep'], driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertNotIn('remove', read)
# Test using append.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2])
for idx in range(3):
sub = field[{'points': idx}]
if idx == 0:
write_mode = MPIWriteMode.WRITE
else:
write_mode = MPIWriteMode.APPEND
sub.write(path, write_mode=write_mode, driver=DriverVector)
self.assertOGRFileLength(path, idx + 1)
def run_create_merged_weight_file(self, filemode):
import ESMF
esmf_filemode = getattr(ESMF.FileMode, filemode)
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0,
wrapped=False,
crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0,
wrapped=False,
crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
src_rd = RequestDataset(path_src, driver='netcdf-cf')
dst_rd = RequestDataset(path_dst, driver='netcdf-cf')
gs = GridChunker(src_rd,
dst_rd, (2, 2),
check_contains=False,
allow_masked=True,
paths=self.fixture_paths,
genweights=True,
filemode=filemode)
gs.write_chunks()
if filemode == "WITHAUX":
weightfile = self.get_temporary_file_path('esmf_weights_1.nc')
vc = RequestDataset(weightfile, driver='netcdf').create_field()
self.assertGreater(len(vc.keys()), 3)
weightfile = self.get_temporary_file_path('esmf_weights_2.nc')
vc = RequestDataset(weightfile, driver='netcdf').get()
self.assertEqual(len(vc.keys()), 3)
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path(
'merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
nvars = len(
RequestDataset(merged_weight_filename,
driver='netcdf').get().keys())
if filemode == "WITHAUX":
self.assertGreater(nvars, 3)
else:
self.assertEqual(nvars, 3)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path(
'global_weights.nc')
srcgrid = ESMF.Grid(filename=path_src,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=path_dst,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield,
dstfield=dstfield,
filename=global_weights_filename,
regrid_method=ESMF.RegridMethod.CONSERVE,
filemode=esmf_filemode,
src_file=path_src,
dst_file=path_dst,
src_file_type=ESMF.FileFormat.GRIDSPEC,
dst_file_type=ESMF.FileFormat.GRIDSPEC)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename,
merged_weight_filename)
