def test_system_grid_chunking(self):
if vm.size != 4:
raise SkipTest('vm.size != 4')
from ocgis.spatial.grid_chunker import GridChunker
path = self.path_esmf_unstruct
rd_dst = RequestDataset(uri=path,
driver=DriverESMFUnstruct,
crs=Spherical(),
grid_abstraction='point',
grid_is_isomorphic=True)
rd_src = deepcopy(rd_dst)
resolution = 0.28125
chunk_wd = os.path.join(self.current_dir_output, 'chunks')
if vm.rank == 0:
os.mkdir(chunk_wd)
vm.barrier()
paths = {'wd': chunk_wd}
gc = GridChunker(rd_src,
rd_dst,
nchunks_dst=[8],
src_grid_resolution=resolution,
dst_grid_resolution=resolution,
optimized_bbox_subset=True,
paths=paths,
genweights=True)
gc.write_chunks()
dist = OcgDist()
local_ctr = Dimension(name='ctr', size=8, dist=True)
dist.add_dimension(local_ctr)
dist.update_dimension_bounds()
for ctr in range(local_ctr.bounds_local[0], local_ctr.bounds_local[1]):
ctr += 1
s = os.path.join(chunk_wd, 'split_src_{}.nc'.format(ctr))
d = os.path.join(chunk_wd, 'split_dst_{}.nc'.format(ctr))
sf = Field.read(s, driver=DriverESMFUnstruct)
df = Field.read(d, driver=DriverESMFUnstruct)
self.assertGreater(sf.grid.shape[0], df.grid.shape[0])
wgt = os.path.join(chunk_wd, 'esmf_weights_{}.nc'.format(ctr))
f = Field.read(wgt)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
with vm.scoped('merge weights', [0]):
if not vm.is_null:
merged_weights = self.get_temporary_file_path(
'merged_weights.nc')
gc.create_merged_weight_file(merged_weights, strict=False)
f = Field.read(merged_weights)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
def test_system_grid_chunking(self):
if vm.size != 4: raise SkipTest('vm.size != 4')
from ocgis.spatial.grid_chunker import GridChunker
path = self.path_esmf_unstruct
rd_dst = RequestDataset(uri=path,
driver=DriverESMFUnstruct,
crs=Spherical(),
grid_abstraction='point',
grid_is_isomorphic=True)
rd_src = deepcopy(rd_dst)
resolution = 0.28125
chunk_wd = os.path.join(self.current_dir_output, 'chunks')
if vm.rank == 0:
os.mkdir(chunk_wd)
vm.barrier()
paths = {'wd': chunk_wd}
gc = GridChunker(rd_src, rd_dst, nchunks_dst=[8], src_grid_resolution=resolution,
dst_grid_resolution=resolution,
optimized_bbox_subset=True, paths=paths, genweights=True)
gc.write_chunks()
dist = OcgDist()
local_ctr = Dimension(name='ctr', size=8, dist=True)
dist.add_dimension(local_ctr)
dist.update_dimension_bounds()
for ctr in range(local_ctr.bounds_local[0], local_ctr.bounds_local[1]):
ctr += 1
s = os.path.join(chunk_wd, 'split_src_{}.nc'.format(ctr))
d = os.path.join(chunk_wd, 'split_dst_{}.nc'.format(ctr))
sf = Field.read(s, driver=DriverESMFUnstruct)
df = Field.read(d, driver=DriverESMFUnstruct)
self.assertLessEqual(sf.grid.shape[0] - df.grid.shape[0], 150)
self.assertGreater(sf.grid.shape[0], df.grid.shape[0])
wgt = os.path.join(chunk_wd, 'esmf_weights_{}.nc'.format(ctr))
f = Field.read(wgt)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
with vm.scoped('merge weights', [0]):
if not vm.is_null:
merged_weights = self.get_temporary_file_path('merged_weights.nc')
gc.create_merged_weight_file(merged_weights, strict=False)
f = Field.read(merged_weights)
S = f['S'].v()
self.assertAlmostEqual(S.min(), 1.0)
self.assertAlmostEqual(S.max(), 1.0)
def fixture(self):
u = get_ugrid_data_structure()
u = Field.from_variable_collection(u)
x = u['face_node_x']
y = u['face_node_y']
cindex = u['face_node_index']
poly = PolygonGC(x, y, cindex=cindex, parent=u)
return poly
def fixture(self):
u = get_ugrid_data_structure()
u = Field.from_variable_collection(u)
x = u['face_node_x']
y = u['face_node_y']
cindex = u['face_node_index']
poly = PolygonGC(x, y, cindex=cindex, parent=u)
return poly
def get_geometryvariable_with_parent():
vpa = np.array([None, None, None])
vpa[:] = [Point(1, 2), Point(3, 4), Point(5, 6)]
value = np.arange(0, 30).reshape(10, 3)
tas = Variable(name='tas', value=value, dimensions=['time', 'ngeom'])
backref = Field(variables=[tas])
pa = GeometryVariable(value=vpa,
parent=backref,
name='point',
dimensions='ngeom')
backref[pa.name] = pa
return pa
def fixture_driver_scrip_netcdf_field(self):
xvalue = np.arange(10., 35., step=5)
yvalue = np.arange(45., 85., step=10)
grid_size = xvalue.shape[0] * yvalue.shape[0]
dim_grid_size = Dimension(name='grid_size', size=grid_size)
x = Variable(name='grid_center_lon', dimensions=dim_grid_size)
y = Variable(name='grid_center_lat', dimensions=dim_grid_size)
for idx, (xv, yv) in enumerate(itertools.product(xvalue, yvalue)):
x.get_value()[idx] = xv
y.get_value()[idx] = yv
gc = PointGC(x=x, y=y, crs=Spherical(), driver=DriverNetcdfSCRIP)
grid = GridUnstruct(geoms=[gc])
ret = Field(grid=grid, driver=DriverNetcdfSCRIP)
grid_dims = Variable(name='grid_dims', value=[yvalue.shape[0], xvalue.shape[0]], dimensions='grid_rank')
ret.add_variable(grid_dims)
return ret
def test_init(self):
# Test empty.
gvar = GeometryVariable()
self.assertEqual(gvar.dtype, object)
gvar = self.get_geometryvariable()
self.assertIsInstance(gvar.get_masked_value(), MaskedArray)
self.assertEqual(gvar.ndim, 1)
# The geometry variable should set itself as the representative geometry on its parent field if that parent does
# not have a representative geometry set.
self.assertIsNotNone(gvar.parent.geom)
# Test with a parent that already has a geometry.
field = Field()
field.set_geom(GeometryVariable(name='empty'))
gvar = self.get_geometryvariable(parent=field)
self.assertEqual(field.geom.name, 'empty')
self.assertIn(gvar.name, field)
# Test passing a "crs".
gvar = self.get_geometryvariable(crs=WGS84(),
name='my_geom',
dimensions='ngeom')
self.assertEqual(gvar.crs, WGS84())
# Test using lines.
line1 = LineString([(0, 0), (1, 1)])
line2 = LineString([(1, 1), (2, 2)])
gvar = GeometryVariable(value=[line1, line2], dimensions='two')
self.assertTrue(gvar.get_value()[1].almost_equals(line2))
self.assertEqual(gvar.geom_type, line1.geom_type)
lines = MultiLineString([line1, line2])
lines2 = [lines, lines]
for actual in [lines, lines2, lines]:
gvar2 = GeometryVariable(value=actual, dimensions='ngeom')
self.assertTrue(gvar2.get_value()[0].almost_equals(lines))
self.assertEqual(gvar2.geom_type, lines.geom_type)
self.assertTrue(gvar2.shape[0] > 0)
self.assertIsNone(gvar2.get_mask())
def test_reduce_global(self):
pt = self.fixture(cindex=self.fixture_cindex(1), start_index=1)
self.assertEqual(pt.start_index, 1)
dist = OcgDist()
for d in pt.parent.dimensions.values():
d = d.copy()
if d.name == self.fixture_element_dimension.name:
d.dist = True
dist.add_dimension(d)
dist.update_dimension_bounds()
new_parent = variable_collection_scatter(pt.parent, dist)
vm.create_subcomm_by_emptyable('coordinate reduction',
new_parent,
is_current=True)
if vm.is_null:
return
pt.parent = new_parent
sub = pt.get_distributed_slice(slice(2, 5))
vm.create_subcomm_by_emptyable('distributed slice',
sub,
is_current=True)
if vm.is_null:
return
actual = sub.reduce_global()
actual_cindex = actual.cindex.extract()
actual_cindex = variable_gather(actual_cindex)
if vm.rank == 0:
actual_cindex = actual_cindex.get_value().flatten().tolist()
self.assertEqual(actual_cindex, [1, 2, 3])
gathered = [
variable_gather(c.extract()) for c in actual.coordinate_variables
]
if vm.rank == 0:
actual_coords = []
for c in gathered:
actual_coords.append(c.get_value().tolist())
desired = [[2.0, 3.0, 4.0], [8.0, 9.0, 10.0], [14.0, 15.0, 16.0]]
self.assertEqual(actual_coords, desired)
path = self.get_temporary_file_path('foo.nc')
actual.parent.write(path)
actual = Field.read(path)
self.assertEqual(actual['cindex'].attrs['start_index'], 1)
def test_init_dimension_map(self):
"""Test initializing with a dimension map only."""
dmap = DimensionMap()
x = Variable(value=[1, 2, 3], dimensions='elements', name='x')
y = Variable(value=[4, 5, 6], dimensions='elements', name='y')
topo = dmap.get_topology(Topology.POINT, create=True)
topo.set_variable(DMK.X, x)
topo.set_variable(DMK.Y, y)
f = Field(variables=[x, y], dimension_map=dmap)
p = PointGC(parent=f)
self.assertNumpyAll(x.get_value(), p.x.get_value())
self.assertNumpyAll(y.get_value(), p.y.get_value())
def test_init(self):
# Test empty.
gvar = GeometryVariable()
self.assertEqual(gvar.dtype, object)
gvar = self.get_geometryvariable()
self.assertIsInstance(gvar.get_masked_value(), MaskedArray)
self.assertEqual(gvar.ndim, 1)
# The geometry variable should set itself as the representative geometry on its parent field if that parent does
# not have a representative geometry set.
self.assertIsNotNone(gvar.parent.geom)
# Test with a parent that already has a geometry.
field = Field()
field.set_geom(GeometryVariable(name='empty'))
gvar = self.get_geometryvariable(parent=field)
self.assertEqual(field.geom.name, 'empty')
self.assertIn(gvar.name, field)
# Test passing a "crs".
gvar = self.get_geometryvariable(crs=WGS84(), name='my_geom', dimensions='ngeom')
self.assertEqual(gvar.crs, WGS84())
# Test using lines.
line1 = LineString([(0, 0), (1, 1)])
line2 = LineString([(1, 1), (2, 2)])
gvar = GeometryVariable(value=[line1, line2], dimensions='two')
self.assertTrue(gvar.get_value()[1].almost_equals(line2))
self.assertEqual(gvar.geom_type, line1.geom_type)
lines = MultiLineString([line1, line2])
lines2 = [lines, lines]
for actual in [lines, lines2, lines]:
gvar2 = GeometryVariable(value=actual, dimensions='ngeom')
self.assertTrue(gvar2.get_value()[0].almost_equals(lines))
self.assertEqual(gvar2.geom_type, lines.geom_type)
self.assertTrue(gvar2.shape[0] > 0)
self.assertIsNone(gvar2.get_mask())
def fixture_driver_scrip_netcdf_field(self):
xvalue = np.arange(10., 35., step=5)
yvalue = np.arange(45., 85., step=10)
grid_size = xvalue.shape[0] * yvalue.shape[0]
dim_grid_size = Dimension(name='grid_size', size=grid_size)
x = Variable(name='grid_center_lon', dimensions=dim_grid_size)
y = Variable(name='grid_center_lat', dimensions=dim_grid_size)
for idx, (xv, yv) in enumerate(itertools.product(xvalue, yvalue)):
x.get_value()[idx] = xv
y.get_value()[idx] = yv
gc = PointGC(x=x, y=y, crs=Spherical(), driver=DriverNetcdfSCRIP)
grid = GridUnstruct(geoms=[gc])
ret = Field(grid=grid, driver=DriverNetcdfSCRIP)
grid_dims = Variable(name='grid_dims',
value=[yvalue.shape[0], xvalue.shape[0]],
dimensions='grid_rank')
ret.add_variable(grid_dims)
return ret
def test_reduce_global(self):
pt = self.fixture(cindex=self.fixture_cindex(1), start_index=1)
self.assertEqual(pt.start_index, 1)
dist = OcgDist()
for d in pt.parent.dimensions.values():
d = d.copy()
if d.name == self.fixture_element_dimension.name:
d.dist = True
dist.add_dimension(d)
dist.update_dimension_bounds()
new_parent = variable_collection_scatter(pt.parent, dist)
vm.create_subcomm_by_emptyable('coordinate reduction', new_parent, is_current=True)
if vm.is_null:
return
pt.parent = new_parent
sub = pt.get_distributed_slice(slice(2, 5))
vm.create_subcomm_by_emptyable('distributed slice', sub, is_current=True)
if vm.is_null:
return
actual = sub.reduce_global()
actual_cindex = actual.cindex.extract()
actual_cindex = variable_gather(actual_cindex)
if vm.rank == 0:
actual_cindex = actual_cindex.get_value().flatten().tolist()
self.assertEqual(actual_cindex, [1, 2, 3])
gathered = [variable_gather(c.extract()) for c in actual.coordinate_variables]
if vm.rank == 0:
actual_coords = []
for c in gathered:
actual_coords.append(c.get_value().tolist())
desired = [[2.0, 3.0, 4.0], [8.0, 9.0, 10.0], [14.0, 15.0, 16.0]]
self.assertEqual(actual_coords, desired)
path = self.get_temporary_file_path('foo.nc')
actual.parent.write(path)
actual = Field.read(path)
self.assertEqual(actual['cindex'].attrs['start_index'], 1)
def test_create_dimension_map(self):
f = self.fixture_esmf_unstruct_field()
for target in [Topology.POINT, Topology.POLYGON]:
topo = f.dimension_map.get_topology(target)
self.assertEqual(topo.get_variable(DMK.X), topo.get_variable(DMK.Y))
f.grid.parent.load()
for target in [Topology.POINT, Topology.POLYGON]:
topo = f.dimension_map.get_topology(target)
self.assertNotEqual(topo.get_variable(DMK.X), topo.get_variable(DMK.Y))
self.assertIn(Topology.POINT, f.grid.abstractions_available)
# Test something in the spatial mask
self.assertTrue(f.grid.has_mask)
self.assertIsNotNone(f.dimension_map.get_spatial_mask())
path = self.get_temporary_file_path('foo.nc')
f.write(path)
af = Field.read(uri=path, driver=DriverESMFUnstruct)
af.grid.parent.load()
self.assertTrue(af.grid.has_mask)
def test_create_dimension_map(self):
f = self.fixture_esmf_unstruct_field()
for target in [Topology.POINT, Topology.POLYGON]:
topo = f.dimension_map.get_topology(target)
self.assertEqual(topo.get_variable(DMK.X), topo.get_variable(DMK.Y))
f.grid.parent.load()
for target in [Topology.POINT, Topology.POLYGON]:
topo = f.dimension_map.get_topology(target)
self.assertNotEqual(topo.get_variable(DMK.X), topo.get_variable(DMK.Y))
self.assertIn(Topology.POINT, f.grid.abstractions_available)
# Test something in the spatial mask
self.assertTrue(f.grid.has_mask)
self.assertIsNotNone(f.dimension_map.get_spatial_mask())
path = self.get_temporary_file_path('foo.nc')
f.write(path)
af = Field.read(uri=path, driver=DriverESMFUnstruct)
af.grid.parent.load()
self.assertTrue(af.grid.has_mask)