def main():
rd = RequestDataset(IN_PATH,
driver=DriverNetcdfUGRID,
grid_abstraction=GridAbstraction.POINT)
field = rd.get()
foo = '/tmp/foo.nc'
# assert field.grid.cindex is not None
# print field.grid.archetype
# tkk
print field.shapes
sub = field.grid.get_intersects(box(*BBOX),
optimized_bbox_subset=True).parent
with vm.scoped_by_emptyable('reduce global', sub):
if not vm.is_null:
sub.grid_abstraction = GridAbstraction.POLYGON
# rank_print('sub.grid.abstraction', sub.grid.abstraction)
# rank_print('sub.grid._abstraction', sub.grid._abstraction)
# rank_print('archetype', sub.grid.archetype)
# rank_print(sub.grid.extent)
rank_print('sub', sub.grid.cindex.get_value())
subr = sub.grid.reduce_global().parent
rank_print('sub', subr.grid.cindex.get_value())
# rank_print(subr.x.name)
# rank_print(subr.x.get_value().min())
rank_print(subr.grid.extent)
# rank_print(subr.grid.cindex.get_value())
# rank_print(subr.shapes)
# subr.write(foo)
# if vm.rank == 0:
# RequestDataset(foo).inspect()
vm.barrier()
def main():
rd = RequestDataset(IN_PATH, driver=DriverNetcdfUGRID, grid_abstraction=GridAbstraction.POINT)
field = rd.get()
foo = '/tmp/foo.nc'
# assert field.grid.cindex is not None
# print field.grid.archetype
# tkk
print field.shapes
sub = field.grid.get_intersects(box(*BBOX), optimized_bbox_subset=True).parent
with vm.scoped_by_emptyable('reduce global', sub):
if not vm.is_null:
sub.grid_abstraction = GridAbstraction.POLYGON
# rank_print('sub.grid.abstraction', sub.grid.abstraction)
# rank_print('sub.grid._abstraction', sub.grid._abstraction)
# rank_print('archetype', sub.grid.archetype)
# rank_print(sub.grid.extent)
rank_print('sub', sub.grid.cindex.get_value())
subr = sub.grid.reduce_global().parent
rank_print('sub', subr.grid.cindex.get_value())
# rank_print(subr.x.name)
# rank_print(subr.x.get_value().min())
rank_print(subr.grid.extent)
# rank_print(subr.grid.cindex.get_value())
# rank_print(subr.shapes)
# subr.write(foo)
# if vm.rank == 0:
# RequestDataset(foo).inspect()
vm.barrier()
def test_system_writing_to_netcdf(self):
"""Test coordinate system is retrievable from netCDF format."""
path = self.get_temporary_file_path('crs.nc')
with nc.Dataset(path, 'w') as rootgrp:
WGS84().write_to_rootgrp(rootgrp)
rd = RequestDataset(path)
infield = rd.get()
actual = infield.first()
self.assertEqual(actual, WGS84())
actual_crs = AbstractProj4CRS.load_from_metadata(rd.metadata)
self.assertEqual(actual_crs, WGS84())
def test_system_writing_to_netcdf(self):
"""Test coordinate system is retrievable from netCDF format."""
path = self.get_temporary_file_path('crs.nc')
with nc.Dataset(path, 'w') as rootgrp:
WGS84().write_to_rootgrp(rootgrp)
rd = RequestDataset(path)
infield = rd.get()
actual = infield.first()
self.assertEqual(actual, WGS84())
actual_crs = AbstractProj4CRS.load_from_metadata(rd.metadata)
self.assertEqual(actual_crs, WGS84())
def plot_subset():
# src_file = '/home/benkoziol/l/data/ocgis/ugrid-cesm-subsetting/UGRID_1km-merge-10min_HYDRO1K-merge-nomask_c130402.nc'
src_file = '/home/benkoziol/htmp/src_subset_57.nc'
dst_file = '/home/benkoziol/htmp/dst_subset_57.nc'
# the_plt = plot_centers(src_file)
ugrid_corner_plotting(src_file, show=False)
rd = RequestDataset(dst_file)
vc = rd.get()
x = vc['x'].get_value()
y = vc['y'].get_value()
x, y = np.meshgrid(x, y)
x = x.flatten()
y = y.flatten()
plt.scatter(x, y, marker='x', color='r')
plt.show()
def plot_subset():
# src_file = '/home/benkoziol/l/data/ocgis/ugrid-cesm-subsetting/UGRID_1km-merge-10min_HYDRO1K-merge-nomask_c130402.nc'
src_file = '/home/benkoziol/htmp/src_subset_57.nc'
dst_file = '/home/benkoziol/htmp/dst_subset_57.nc'
# the_plt = plot_centers(src_file)
ugrid_corner_plotting(src_file, show=False)
rd = RequestDataset(dst_file)
vc = rd.get()
x = vc['x'].get_value()
y = vc['y'].get_value()
x, y = np.meshgrid(x, y)
x = x.flatten()
y = y.flatten()
plt.scatter(x, y, marker='x', color='r')
plt.show()
def test_system_get_field_from_file(self):
"""Test returning a distributed field from file."""
field = self.get_field(nrow=5, ncol=7)
if MPI_RANK == 0:
path = self.get_temporary_file_path('data.nc')
else:
path = None
path = MPI_COMM.bcast(path)
with vm.scoped('write test field', [0]):
if MPI_RANK == 0:
field.write(path)
MPI_COMM.Barrier()
rd = RequestDataset(path)
out_field = rd.get()
if MPI_SIZE == 8:
self.assertEqual(vm.size, 8)
if MPI_RANK == 0:
path2 = self.get_temporary_file_path('out_field.nc')
else:
path2 = None
path2 = MPI_COMM.bcast(path2)
with vm.scoped_by_emptyable('out_field write', out_field):
if not vm.is_null:
out_field.write(path2)
MPI_COMM.Barrier()
with vm.scoped('get actual', [0]):
if MPI_RANK == 0:
actual = RequestDataset(path2).get()
actual = actual.data_variables[0].get_value().sum()
else:
actual = None
actual = MPI_COMM.bcast(actual)
desired = field.data_variables[0].get_value().sum()
self.assertAlmostEqual(actual, desired)
def test_system_get_field_from_file(self):
"""Test returning a distributed field from file."""
field = self.get_field(nrow=5, ncol=7)
if MPI_RANK == 0:
path = self.get_temporary_file_path('data.nc')
else:
path = None
path = MPI_COMM.bcast(path)
with vm.scoped('write test field', [0]):
if MPI_RANK == 0:
field.write(path)
MPI_COMM.Barrier()
rd = RequestDataset(path)
out_field = rd.get()
if MPI_SIZE == 8:
self.assertEqual(vm.size, 8)
if MPI_RANK == 0:
path2 = self.get_temporary_file_path('out_field.nc')
else:
path2 = None
path2 = MPI_COMM.bcast(path2)
with vm.scoped_by_emptyable('out_field write', out_field):
if not vm.is_null:
out_field.write(path2)
MPI_COMM.Barrier()
with vm.scoped('get actual', [0]):
if MPI_RANK == 0:
actual = RequestDataset(path2).get()
actual = actual.data_variables[0].get_value().sum()
else:
actual = None
actual = MPI_COMM.bcast(actual)
desired = field.data_variables[0].get_value().sum()
self.assertAlmostEqual(actual, desired)
def test_system_converting_state_boundaries_shapefile_memory(self):
"""Test iteration may be used in place of loading all values from source."""
rd = RequestDataset(uri=self.path_state_boundaries)
field = rd.get()
data_variable_names = get_variable_names(field.data_variables)
field.geom.protected = True
sub = field.get_field_slice({'geom': slice(10, 20)})
self.assertTrue(sub.geom.protected)
self.assertFalse(sub.geom.has_allocated_value)
self.assertIsInstance(sub, Field)
self.assertIsInstance(sub.geom, GeometryVariable)
gc = sub.geom.convert_to(use_geometry_iterator=True)
self.assertIsInstance(gc, PolygonGC)
self.assertFalse(sub.geom.has_allocated_value)
self.assertTrue(field.geom.protected)
path = self.get_temporary_file_path('out.nc')
gc.parent.write(path)
def test_system_converting_state_boundaries_shapefile_memory(self):
"""Test iteration may be used in place of loading all values from source."""
rd = RequestDataset(uri=self.path_state_boundaries)
field = rd.get()
data_variable_names = get_variable_names(field.data_variables)
field.geom.protected = True
sub = field.get_field_slice({'geom': slice(10, 20)})
self.assertTrue(sub.geom.protected)
self.assertFalse(sub.geom.has_allocated_value)
self.assertIsInstance(sub, Field)
self.assertIsInstance(sub.geom, GeometryVariable)
gc = sub.geom.convert_to(use_geometry_iterator=True)
self.assertIsInstance(gc, PolygonGC)
# Test the new object does not share data with the source.
for dn in data_variable_names:
self.assertNotIn(dn, gc.parent)
self.assertFalse(sub.geom.has_allocated_value)
self.assertTrue(field.geom.protected)
path = self.get_temporary_file_path('out.nc')
gc.parent.write(path)
def test_system_converting_state_boundaries_shapefile(self):
ocgis.env.USE_NETCDF4_MPI = False # tdk:FIX: this hangs in the STATE_FIPS write for asynch might be nc4 bug...
keywords = {'transform_to_crs': [None, Spherical],
'use_geometry_iterator': [False, True]}
actual_xsums = []
actual_ysums = []
for k in self.iter_product_keywords(keywords):
if k.use_geometry_iterator and k.transform_to_crs is not None:
to_crs = k.transform_to_crs()
else:
to_crs = None
if k.transform_to_crs is None:
desired_crs = WGS84()
else:
desired_crs = k.transform_to_crs()
rd = RequestDataset(uri=self.path_state_boundaries, variable=['UGID', 'ID'])
rd.metadata['schema']['geometry'] = 'MultiPolygon'
field = rd.get()
self.assertEqual(len(field.data_variables), 2)
# Test there is no mask present.
field.geom.load()
self.assertFalse(field.geom.has_mask)
self.assertNotIn(VariableName.SPATIAL_MASK, field)
self.assertIsNone(field.dimension_map.get_spatial_mask())
self.assertEqual(field.crs, WGS84())
if k.transform_to_crs is not None:
field.update_crs(desired_crs)
self.assertEqual(len(field.data_variables), 2)
self.assertEqual(len(field.geom.parent.data_variables), 2)
try:
gc = field.geom.convert_to(pack=False, use_geometry_iterator=k.use_geometry_iterator, to_crs=to_crs)
except ValueError as e:
try:
self.assertFalse(k.use_geometry_iterator)
self.assertIsNotNone(to_crs)
except AssertionError:
raise e
else:
continue
actual_xsums.append(gc.x.get_value().sum())
actual_ysums.append(gc.y.get_value().sum())
self.assertEqual(gc.crs, desired_crs)
# Test there is no mask present after conversion to geometry coordinates.
self.assertFalse(gc.has_mask)
self.assertNotIn(VariableName.SPATIAL_MASK, gc.parent)
self.assertIsNone(gc.dimension_map.get_spatial_mask())
path = self.get_temporary_file_path('esmf_state_boundaries.nc')
self.assertEqual(gc.parent.crs, desired_crs)
gc.parent.write(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT)
gathered_geoms = vm.gather(field.geom.get_value())
if vm.rank == 0:
actual_geoms = []
for g in gathered_geoms:
actual_geoms.extend(g)
rd = RequestDataset(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT)
infield = rd.get()
self.assertEqual(create_crs(infield.crs.value), desired_crs)
for dv in field.data_variables:
self.assertIn(dv.name, infield)
ingrid = infield.grid
self.assertIsInstance(ingrid, GridUnstruct)
for g in ingrid.archetype.iter_geometries():
self.assertPolygonSimilar(g[1], actual_geoms[g[0]], check_type=False)
vm.barrier()
# Test coordinates have actually changed.
if not k.use_geometry_iterator:
for ctr, to_test in enumerate([actual_xsums, actual_ysums]):
for lhs, rhs in itertools.combinations(to_test, 2):
if ctr == 0:
self.assertAlmostEqual(lhs, rhs)
else:
self.assertNotAlmostEqual(lhs, rhs)
def test_system_converting_state_boundaries_shapefile(self):
ocgis.env.USE_NETCDF4_MPI = False # tdk:FIX: this hangs in the STATE_FIPS write for asynch might be nc4 bug...
keywords = {'transform_to_crs': [None, Spherical],
'use_geometry_iterator': [False, True]}
actual_xsums = []
actual_ysums = []
for k in self.iter_product_keywords(keywords):
if k.use_geometry_iterator and k.transform_to_crs is not None:
to_crs = k.transform_to_crs()
else:
to_crs = None
if k.transform_to_crs is None:
desired_crs = WGS84()
else:
desired_crs = k.transform_to_crs()
rd = RequestDataset(uri=self.path_state_boundaries)
rd.metadata['schema']['geometry'] = 'MultiPolygon'
field = rd.get()
# Test there is no mask present.
field.geom.load()
self.assertFalse(field.geom.has_mask)
self.assertNotIn(VariableName.SPATIAL_MASK, field)
self.assertIsNone(field.dimension_map.get_spatial_mask())
self.assertEqual(field.crs, WGS84())
if k.transform_to_crs is not None:
field.update_crs(desired_crs)
try:
gc = field.geom.convert_to(pack=False, use_geometry_iterator=k.use_geometry_iterator, to_crs=to_crs)
except ValueError as e:
try:
self.assertFalse(k.use_geometry_iterator)
self.assertIsNotNone(to_crs)
except AssertionError:
raise e
else:
continue
actual_xsums.append(gc.x.get_value().sum())
actual_ysums.append(gc.y.get_value().sum())
self.assertEqual(gc.crs, desired_crs)
# Test there is no mask present after conversion to geometry coordinates.
self.assertFalse(gc.has_mask)
self.assertNotIn(VariableName.SPATIAL_MASK, gc.parent)
self.assertIsNone(gc.dimension_map.get_spatial_mask())
for v in list(field.values()):
if v.name != field.geom.name:
gc.parent.add_variable(v.extract(), force=True)
path = self.get_temporary_file_path('esmf_state_boundaries.nc')
self.assertEqual(gc.parent.crs, desired_crs)
gc.parent.write(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT)
gathered_geoms = vm.gather(field.geom.get_value())
if vm.rank == 0:
actual_geoms = []
for g in gathered_geoms:
actual_geoms.extend(g)
rd = RequestDataset(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT)
infield = rd.get()
self.assertEqual(create_crs(infield.crs.value), desired_crs)
for dv in field.data_variables:
self.assertIn(dv.name, infield)
ingrid = infield.grid
self.assertIsInstance(ingrid, GridUnstruct)
for g in ingrid.archetype.iter_geometries():
self.assertPolygonSimilar(g[1], actual_geoms[g[0]], check_type=False)
vm.barrier()
# Test coordinates have actually changed.
if not k.use_geometry_iterator:
for ctr, to_test in enumerate([actual_xsums, actual_ysums]):
for lhs, rhs in itertools.combinations(to_test, 2):
if ctr == 0:
self.assertAlmostEqual(lhs, rhs)
else:
self.assertNotAlmostEqual(lhs, rhs)