def main(): rd = RequestDataset(IN_PATH, driver=DriverNetcdfUGRID, grid_abstraction=GridAbstraction.POINT) field = rd.get() foo = '/tmp/foo.nc' # assert field.grid.cindex is not None # print field.grid.archetype # tkk print field.shapes sub = field.grid.get_intersects(box(*BBOX), optimized_bbox_subset=True).parent with vm.scoped_by_emptyable('reduce global', sub): if not vm.is_null: sub.grid_abstraction = GridAbstraction.POLYGON # rank_print('sub.grid.abstraction', sub.grid.abstraction) # rank_print('sub.grid._abstraction', sub.grid._abstraction) # rank_print('archetype', sub.grid.archetype) # rank_print(sub.grid.extent) rank_print('sub', sub.grid.cindex.get_value()) subr = sub.grid.reduce_global().parent rank_print('sub', subr.grid.cindex.get_value()) # rank_print(subr.x.name) # rank_print(subr.x.get_value().min()) rank_print(subr.grid.extent) # rank_print(subr.grid.cindex.get_value()) # rank_print(subr.shapes) # subr.write(foo) # if vm.rank == 0: # RequestDataset(foo).inspect() vm.barrier()
def main(): rd = RequestDataset(IN_PATH, driver=DriverNetcdfUGRID, grid_abstraction=GridAbstraction.POINT) field = rd.get() foo = '/tmp/foo.nc' # assert field.grid.cindex is not None # print field.grid.archetype # tkk print field.shapes sub = field.grid.get_intersects(box(*BBOX), optimized_bbox_subset=True).parent with vm.scoped_by_emptyable('reduce global', sub): if not vm.is_null: sub.grid_abstraction = GridAbstraction.POLYGON # rank_print('sub.grid.abstraction', sub.grid.abstraction) # rank_print('sub.grid._abstraction', sub.grid._abstraction) # rank_print('archetype', sub.grid.archetype) # rank_print(sub.grid.extent) rank_print('sub', sub.grid.cindex.get_value()) subr = sub.grid.reduce_global().parent rank_print('sub', subr.grid.cindex.get_value()) # rank_print(subr.x.name) # rank_print(subr.x.get_value().min()) rank_print(subr.grid.extent) # rank_print(subr.grid.cindex.get_value()) # rank_print(subr.shapes) # subr.write(foo) # if vm.rank == 0: # RequestDataset(foo).inspect() vm.barrier()
def test_system_writing_to_netcdf(self): """Test coordinate system is retrievable from netCDF format.""" path = self.get_temporary_file_path('crs.nc') with nc.Dataset(path, 'w') as rootgrp: WGS84().write_to_rootgrp(rootgrp) rd = RequestDataset(path) infield = rd.get() actual = infield.first() self.assertEqual(actual, WGS84()) actual_crs = AbstractProj4CRS.load_from_metadata(rd.metadata) self.assertEqual(actual_crs, WGS84())
def test_system_writing_to_netcdf(self): """Test coordinate system is retrievable from netCDF format.""" path = self.get_temporary_file_path('crs.nc') with nc.Dataset(path, 'w') as rootgrp: WGS84().write_to_rootgrp(rootgrp) rd = RequestDataset(path) infield = rd.get() actual = infield.first() self.assertEqual(actual, WGS84()) actual_crs = AbstractProj4CRS.load_from_metadata(rd.metadata) self.assertEqual(actual_crs, WGS84())
def plot_subset(): # src_file = '/home/benkoziol/l/data/ocgis/ugrid-cesm-subsetting/UGRID_1km-merge-10min_HYDRO1K-merge-nomask_c130402.nc' src_file = '/home/benkoziol/htmp/src_subset_57.nc' dst_file = '/home/benkoziol/htmp/dst_subset_57.nc' # the_plt = plot_centers(src_file) ugrid_corner_plotting(src_file, show=False) rd = RequestDataset(dst_file) vc = rd.get() x = vc['x'].get_value() y = vc['y'].get_value() x, y = np.meshgrid(x, y) x = x.flatten() y = y.flatten() plt.scatter(x, y, marker='x', color='r') plt.show()
def plot_subset(): # src_file = '/home/benkoziol/l/data/ocgis/ugrid-cesm-subsetting/UGRID_1km-merge-10min_HYDRO1K-merge-nomask_c130402.nc' src_file = '/home/benkoziol/htmp/src_subset_57.nc' dst_file = '/home/benkoziol/htmp/dst_subset_57.nc' # the_plt = plot_centers(src_file) ugrid_corner_plotting(src_file, show=False) rd = RequestDataset(dst_file) vc = rd.get() x = vc['x'].get_value() y = vc['y'].get_value() x, y = np.meshgrid(x, y) x = x.flatten() y = y.flatten() plt.scatter(x, y, marker='x', color='r') plt.show()
def test_system_get_field_from_file(self): """Test returning a distributed field from file.""" field = self.get_field(nrow=5, ncol=7) if MPI_RANK == 0: path = self.get_temporary_file_path('data.nc') else: path = None path = MPI_COMM.bcast(path) with vm.scoped('write test field', [0]): if MPI_RANK == 0: field.write(path) MPI_COMM.Barrier() rd = RequestDataset(path) out_field = rd.get() if MPI_SIZE == 8: self.assertEqual(vm.size, 8) if MPI_RANK == 0: path2 = self.get_temporary_file_path('out_field.nc') else: path2 = None path2 = MPI_COMM.bcast(path2) with vm.scoped_by_emptyable('out_field write', out_field): if not vm.is_null: out_field.write(path2) MPI_COMM.Barrier() with vm.scoped('get actual', [0]): if MPI_RANK == 0: actual = RequestDataset(path2).get() actual = actual.data_variables[0].get_value().sum() else: actual = None actual = MPI_COMM.bcast(actual) desired = field.data_variables[0].get_value().sum() self.assertAlmostEqual(actual, desired)
def test_system_get_field_from_file(self): """Test returning a distributed field from file.""" field = self.get_field(nrow=5, ncol=7) if MPI_RANK == 0: path = self.get_temporary_file_path('data.nc') else: path = None path = MPI_COMM.bcast(path) with vm.scoped('write test field', [0]): if MPI_RANK == 0: field.write(path) MPI_COMM.Barrier() rd = RequestDataset(path) out_field = rd.get() if MPI_SIZE == 8: self.assertEqual(vm.size, 8) if MPI_RANK == 0: path2 = self.get_temporary_file_path('out_field.nc') else: path2 = None path2 = MPI_COMM.bcast(path2) with vm.scoped_by_emptyable('out_field write', out_field): if not vm.is_null: out_field.write(path2) MPI_COMM.Barrier() with vm.scoped('get actual', [0]): if MPI_RANK == 0: actual = RequestDataset(path2).get() actual = actual.data_variables[0].get_value().sum() else: actual = None actual = MPI_COMM.bcast(actual) desired = field.data_variables[0].get_value().sum() self.assertAlmostEqual(actual, desired)
def test_system_converting_state_boundaries_shapefile_memory(self): """Test iteration may be used in place of loading all values from source.""" rd = RequestDataset(uri=self.path_state_boundaries) field = rd.get() data_variable_names = get_variable_names(field.data_variables) field.geom.protected = True sub = field.get_field_slice({'geom': slice(10, 20)}) self.assertTrue(sub.geom.protected) self.assertFalse(sub.geom.has_allocated_value) self.assertIsInstance(sub, Field) self.assertIsInstance(sub.geom, GeometryVariable) gc = sub.geom.convert_to(use_geometry_iterator=True) self.assertIsInstance(gc, PolygonGC) self.assertFalse(sub.geom.has_allocated_value) self.assertTrue(field.geom.protected) path = self.get_temporary_file_path('out.nc') gc.parent.write(path)
def test_system_converting_state_boundaries_shapefile_memory(self): """Test iteration may be used in place of loading all values from source.""" rd = RequestDataset(uri=self.path_state_boundaries) field = rd.get() data_variable_names = get_variable_names(field.data_variables) field.geom.protected = True sub = field.get_field_slice({'geom': slice(10, 20)}) self.assertTrue(sub.geom.protected) self.assertFalse(sub.geom.has_allocated_value) self.assertIsInstance(sub, Field) self.assertIsInstance(sub.geom, GeometryVariable) gc = sub.geom.convert_to(use_geometry_iterator=True) self.assertIsInstance(gc, PolygonGC) # Test the new object does not share data with the source. for dn in data_variable_names: self.assertNotIn(dn, gc.parent) self.assertFalse(sub.geom.has_allocated_value) self.assertTrue(field.geom.protected) path = self.get_temporary_file_path('out.nc') gc.parent.write(path)
def test_system_converting_state_boundaries_shapefile(self): ocgis.env.USE_NETCDF4_MPI = False # tdk:FIX: this hangs in the STATE_FIPS write for asynch might be nc4 bug... keywords = {'transform_to_crs': [None, Spherical], 'use_geometry_iterator': [False, True]} actual_xsums = [] actual_ysums = [] for k in self.iter_product_keywords(keywords): if k.use_geometry_iterator and k.transform_to_crs is not None: to_crs = k.transform_to_crs() else: to_crs = None if k.transform_to_crs is None: desired_crs = WGS84() else: desired_crs = k.transform_to_crs() rd = RequestDataset(uri=self.path_state_boundaries, variable=['UGID', 'ID']) rd.metadata['schema']['geometry'] = 'MultiPolygon' field = rd.get() self.assertEqual(len(field.data_variables), 2) # Test there is no mask present. field.geom.load() self.assertFalse(field.geom.has_mask) self.assertNotIn(VariableName.SPATIAL_MASK, field) self.assertIsNone(field.dimension_map.get_spatial_mask()) self.assertEqual(field.crs, WGS84()) if k.transform_to_crs is not None: field.update_crs(desired_crs) self.assertEqual(len(field.data_variables), 2) self.assertEqual(len(field.geom.parent.data_variables), 2) try: gc = field.geom.convert_to(pack=False, use_geometry_iterator=k.use_geometry_iterator, to_crs=to_crs) except ValueError as e: try: self.assertFalse(k.use_geometry_iterator) self.assertIsNotNone(to_crs) except AssertionError: raise e else: continue actual_xsums.append(gc.x.get_value().sum()) actual_ysums.append(gc.y.get_value().sum()) self.assertEqual(gc.crs, desired_crs) # Test there is no mask present after conversion to geometry coordinates. self.assertFalse(gc.has_mask) self.assertNotIn(VariableName.SPATIAL_MASK, gc.parent) self.assertIsNone(gc.dimension_map.get_spatial_mask()) path = self.get_temporary_file_path('esmf_state_boundaries.nc') self.assertEqual(gc.parent.crs, desired_crs) gc.parent.write(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT) gathered_geoms = vm.gather(field.geom.get_value()) if vm.rank == 0: actual_geoms = [] for g in gathered_geoms: actual_geoms.extend(g) rd = RequestDataset(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT) infield = rd.get() self.assertEqual(create_crs(infield.crs.value), desired_crs) for dv in field.data_variables: self.assertIn(dv.name, infield) ingrid = infield.grid self.assertIsInstance(ingrid, GridUnstruct) for g in ingrid.archetype.iter_geometries(): self.assertPolygonSimilar(g[1], actual_geoms[g[0]], check_type=False) vm.barrier() # Test coordinates have actually changed. if not k.use_geometry_iterator: for ctr, to_test in enumerate([actual_xsums, actual_ysums]): for lhs, rhs in itertools.combinations(to_test, 2): if ctr == 0: self.assertAlmostEqual(lhs, rhs) else: self.assertNotAlmostEqual(lhs, rhs)
def test_system_converting_state_boundaries_shapefile(self): ocgis.env.USE_NETCDF4_MPI = False # tdk:FIX: this hangs in the STATE_FIPS write for asynch might be nc4 bug... keywords = {'transform_to_crs': [None, Spherical], 'use_geometry_iterator': [False, True]} actual_xsums = [] actual_ysums = [] for k in self.iter_product_keywords(keywords): if k.use_geometry_iterator and k.transform_to_crs is not None: to_crs = k.transform_to_crs() else: to_crs = None if k.transform_to_crs is None: desired_crs = WGS84() else: desired_crs = k.transform_to_crs() rd = RequestDataset(uri=self.path_state_boundaries) rd.metadata['schema']['geometry'] = 'MultiPolygon' field = rd.get() # Test there is no mask present. field.geom.load() self.assertFalse(field.geom.has_mask) self.assertNotIn(VariableName.SPATIAL_MASK, field) self.assertIsNone(field.dimension_map.get_spatial_mask()) self.assertEqual(field.crs, WGS84()) if k.transform_to_crs is not None: field.update_crs(desired_crs) try: gc = field.geom.convert_to(pack=False, use_geometry_iterator=k.use_geometry_iterator, to_crs=to_crs) except ValueError as e: try: self.assertFalse(k.use_geometry_iterator) self.assertIsNotNone(to_crs) except AssertionError: raise e else: continue actual_xsums.append(gc.x.get_value().sum()) actual_ysums.append(gc.y.get_value().sum()) self.assertEqual(gc.crs, desired_crs) # Test there is no mask present after conversion to geometry coordinates. self.assertFalse(gc.has_mask) self.assertNotIn(VariableName.SPATIAL_MASK, gc.parent) self.assertIsNone(gc.dimension_map.get_spatial_mask()) for v in list(field.values()): if v.name != field.geom.name: gc.parent.add_variable(v.extract(), force=True) path = self.get_temporary_file_path('esmf_state_boundaries.nc') self.assertEqual(gc.parent.crs, desired_crs) gc.parent.write(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT) gathered_geoms = vm.gather(field.geom.get_value()) if vm.rank == 0: actual_geoms = [] for g in gathered_geoms: actual_geoms.extend(g) rd = RequestDataset(path, driver=DriverKey.NETCDF_ESMF_UNSTRUCT) infield = rd.get() self.assertEqual(create_crs(infield.crs.value), desired_crs) for dv in field.data_variables: self.assertIn(dv.name, infield) ingrid = infield.grid self.assertIsInstance(ingrid, GridUnstruct) for g in ingrid.archetype.iter_geometries(): self.assertPolygonSimilar(g[1], actual_geoms[g[0]], check_type=False) vm.barrier() # Test coordinates have actually changed. if not k.use_geometry_iterator: for ctr, to_test in enumerate([actual_xsums, actual_ysums]): for lhs, rhs in itertools.combinations(to_test, 2): if ctr == 0: self.assertAlmostEqual(lhs, rhs) else: self.assertNotAlmostEqual(lhs, rhs)