def test_create_merged_weight_file(self): import ESMF path_src = self.get_temporary_file_path('src.nc') path_dst = self.get_temporary_file_path('dst.nc') src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical()) dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical()) src_grid.write(path_src) dst_grid.write(path_dst) # Split source and destination grids --------------------------------------------------------------------------- gs = GridChunker(src_grid, dst_grid, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths, genweights=True) gs.write_chunks() # Merge weight files ------------------------------------------------------------------------------------------- merged_weight_filename = self.get_temporary_file_path( 'merged_weights.nc') gs.create_merged_weight_file(merged_weight_filename) # Generate a global weight file using ESMF --------------------------------------------------------------------- global_weights_filename = self.get_temporary_file_path( 'global_weights.nc') srcgrid = ESMF.Grid(filename=path_src, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) dstgrid = ESMF.Grid(filename=path_dst, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) srcfield = ESMF.Field(grid=srcgrid) dstfield = ESMF.Field(grid=dstgrid) _ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=global_weights_filename, regrid_method=ESMF.RegridMethod.CONSERVE) # Test merged and global weight files are equivalent ----------------------------------------------------------- self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_create_merged_weight_file(self): path_src = self.get_temporary_file_path('src.nc') path_dst = self.get_temporary_file_path('dst.nc') src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical()) dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical()) src_grid.write(path_src) dst_grid.write(path_dst) # Split source and destination grids --------------------------------------------------------------------------- gs = GridSplitter(src_grid, dst_grid, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths) gs.write_subsets() # Load the grid splitter index file ---------------------------------------------------------------------------- index_filename = gs.create_full_path_from_template('index_file') ifile = RequestDataset(uri=index_filename).get() ifile.load() gidx = ifile[GridSplitterConstants.IndexFile.NAME_INDEX_VARIABLE].attrs source_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_SOURCE_VARIABLE]] sv = source_filename.join_string_value() destination_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_DESTINATION_VARIABLE]] dv = destination_filename.join_string_value() # Create weight files for each subset -------------------------------------------------------------------------- for ii, sfn in enumerate(sv): esp = os.path.join(self.current_dir_output, sfn) edp = os.path.join(self.current_dir_output, dv[ii]) ewp = gs.create_full_path_from_template('wgt_template', index=ii + 1) cmd = ['ESMF_RegridWeightGen', '-s', esp, '--src_type', 'GRIDSPEC', '-d', edp, '--dst_type', 'GRIDSPEC', '-w', ewp, '--method', 'conserve', '-r', '--no_log'] subprocess.check_call(cmd) # Merge weight files ------------------------------------------------------------------------------------------- merged_weight_filename = self.get_temporary_file_path('merged_weights.nc') gs.create_merged_weight_file(merged_weight_filename) # Generate a global weight file using ESMF --------------------------------------------------------------------- global_weights_filename = self.get_temporary_file_path('global_weights.nc') cmd = ['ESMF_RegridWeightGen', '-s', path_src, '--src_type', 'GRIDSPEC', '-d', path_dst, '--dst_type', 'GRIDSPEC', '-w', global_weights_filename, '--method', 'conserve', '--weight-only', '--no_log'] subprocess.check_call(cmd) # Test merged and global weight files are equivalent ----------------------------------------------------------- self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_system_multiple_netcdf_files(self): """Test subsetting multiple netCDF files and returning a spatial collection.""" grid = create_gridxy_global(resolution=3.0) vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax'] paths = [self.get_temporary_file_path('{}.nc'.format(ii)) for ii in vars] geom_select_uid = [16, 23] field_names = ['tasmin', 'tas', 'tasmax'] for ctr, (path, var) in enumerate(zip(paths, vars), start=1): field = create_exact_field(grid.copy(), var, ntime=3) field.data_variables[0].get_value()[:] = 10 * ctr field.write(path) rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in zip(paths, vars, field_names)] ops = OcgOperations(dataset=rds, spatial_operation='clip', aggregate=True, geom=self.path_state_boundaries, geom_select_uid=geom_select_uid) ret = ops.execute() self.assertAsSetEqual(ret.children.keys(), geom_select_uid) for geom_uid in geom_select_uid: actual = ret.children[geom_uid].children.keys() self.assertAsSetEqual(actual, field_names) for idx, field_name in enumerate(field_names): actual = ret.get_element(container_ugid=geom_uid, field_name=field_names[idx], variable_name=vars[idx]) actual = actual.get_value() actual = actual == (idx + 1) * 10 self.assertTrue(np.all(actual))
def test_iter_spatial_decomposition(self): self.add_barrier = False if vm.size not in [1, 4]: raise SkipTest('vm.size not in [1, 4]') grid = create_gridxy_global(resolution=10., wrapped=False, crs=Spherical()) splits = (2, 3) actual = [] for sub, slc in iter_spatial_decomposition(grid, splits, optimized_bbox_subset=True): root = vm.get_live_ranks_from_object(sub)[0] with vm.scoped_by_emptyable('test extent', sub): if vm.is_null: extent_global = None else: extent_global = sub.extent_global extent_global = vm.bcast(extent_global, root=root) actual.append(extent_global) desired = [(0.0, -90.0, 120.0, 0.0), (120.0, -90.0, 240.0, 0.0), (240.0, -90.0, 360.0, 0.0), (0.0, 0.0, 120.0, 90.0), (120.0, 0.0, 240.0, 90.0), (240.0, 0.0, 360.0, 90.0)] self.assertEqual(actual, desired)
def test_get_ocgis_field_from_esmf_field(self): from ocgis.regrid.base import get_esmf_field_from_ocgis_field from ocgis.regrid.base import get_ocgis_field_from_esmf_field ogrid = create_gridxy_global(crs=Spherical()) ofield = create_exact_field(ogrid, 'foo', ntime=3) ogrid = ofield.grid ogrid_mask = ogrid.get_mask(create=True) ogrid_mask[1, 0] = True ogrid.set_mask(ogrid_mask) efield = get_esmf_field_from_ocgis_field(ofield) self.assertEqual(efield.data.shape, (360, 180, 3)) ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield) actual_dv_mask = ofield_actual.data_variables[0].get_mask() self.assertTrue(np.all(actual_dv_mask[:, 1, 0])) self.assertEqual(actual_dv_mask.sum(), 3) self.assertNumpyAll(ofield.data_variables[0].get_value(), ofield_actual.data_variables[0].get_value()) self.assertEqual(ofield.data_variables[0].name, efield.name) self.assertNumpyAll(ofield.time.get_value(), ofield_actual.time.get_value())
def test_system_masking_with_smm(self): """Test masking with sparse matrix multiplication.""" from ocgis.regrid import RegridOperation grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0) src_field = create_exact_field(grid, 'exact', ntime=3) mask = src_field.grid.get_mask(create=True) mask[0:2, :] = True mask[:, -2:] = True mask[-2:, :] = True mask[:, 0:2] = True src_field.grid.set_mask(mask, cascade=True) src_field['exact'].set_value(src_field['exact'].mv().filled()) dst_field = deepcopy(src_field) dst_field.remove_variable('exact') weights = self.get_temporary_file_path('weights.nc', collective=True) weights = vm.bcast(weights) ro = RegridOperation(src_field, dst_field, regrid_options={'weights_out': weights, 'split': False}) _ = ro.execute() ro2 = RegridOperation(src_field, dst_field, regrid_options={'weights_in': weights, 'split': True}) result = ro2.execute() actual = result['exact'].mv() desired = src_field['exact'].mv() self.assertNumpyAllClose(actual, desired)
def test_system_scrip_destination_splitting(self): """Test splitting a SCRIP destination grid.""" src_grid = create_gridxy_global() dst_grid = self.fixture_driver_scrip_netcdf_field().grid gs = GridChunker(src_grid, dst_grid, (3,), paths={'wd': self.current_dir_output}) gs.write_chunks() self.assertEqual(len(os.listdir(self.current_dir_output)), 7)
def test_transform_coordinates(self): desired_min_maxes = [[-0.9330127018922193, 0.93301270189221941], [-0.93301270189221941, 0.93301270189221941], [-0.96592582628906831, 0.96592582628906831]] keywords = {'wrapped': [ False, True ], 'angular_units': [ OcgisUnits.DEGREES, OcgisUnits.RADIANS ], 'other_crs': [ Cartesian(), WGS84() ] } for k in self.iter_product_keywords(keywords): spherical = Spherical(angular_units=k.angular_units) tp = Tripole(spherical=spherical) grid = create_gridxy_global(resolution=30.0, wrapped=k.wrapped) if not k.wrapped: x_value = grid.x.get_value() select = x_value > 180. x_value[select] -= 360. grid.expand() x = grid.x.get_value() y = grid.y.get_value() if k.angular_units == OcgisUnits.RADIANS: x *= ConversionFactor.DEG_TO_RAD y *= ConversionFactor.DEG_TO_RAD z = np.ones(x.shape, dtype=x.dtype) desired = (x, y, z) try: as_cart = tp.transform_coordinates(k.other_crs, x, y, z) except CRSNotEquivalenError: self.assertNotEqual(k.other_crs, Cartesian()) continue x_cart, y_cart, z_cart = as_cart for idx, ii in enumerate(as_cart): actual_min_max = [ii.min(), ii.max()] self.assertNumpyAllClose(np.array(actual_min_max), np.array(desired_min_maxes[idx])) actual = tp.transform_coordinates(k.other_crs, x_cart, y_cart, z_cart, inverse=True) for a, d in zip(actual, desired): are = np.abs(a - d) self.assertLessEqual(are.max(), 1e-6)
def test_get_esmf_grid_periodicity(self): """Test periodicity parameters generate reasonable output.""" from ocgis.regrid.base import create_esmf_grid ogrid = create_gridxy_global(resolution=10.0, crs=Spherical(), with_bounds=False, dist_dimname='x') egrid = create_esmf_grid(ogrid) self.assertEqual(egrid.periodic_dim, 0) self.assertEqual(egrid.num_peri_dims, 1) self.assertEqual(egrid.pole_dim, 1)
def test_to_xarray(self): grid = create_gridxy_global(crs=Spherical()) field = create_exact_field(grid, 'foo', ntime=3) field.attrs['i_am_global'] = 'confirm' field.grid.abstraction = Topology.POINT field.set_abstraction_geom() field.time.set_extrapolated_bounds('time_bounds', 'bounds') xr = field.to_xarray() self.assertEqual(xr.attrs['i_am_global'], 'confirm') self.assertGreater(len(xr.coords), 0)
def test_system_user_geometry_identifier_typed_appropriately(self): """Test UGID is typed appropriately according to the data model.""" ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'} grid = create_gridxy_global(resolution=3.0) field = create_exact_field(grid, 'foo', crs=Spherical()) ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True) actual = ops.execute() actual = actual[1] self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_system_scrip_destination_splitting(self): """Test splitting a SCRIP destination grid.""" src_grid = create_gridxy_global() dst_grid = self.fixture_driver_scrip_netcdf_field().grid gc = GridChunker(src_grid, dst_grid, (3, ), paths={'wd': self.current_dir_output}) gc.write_chunks() self.assertEqual(len(os.listdir(self.current_dir_output)), 7)
def test_transform_coordinates(self): desired_min_maxes = [[-0.9330127018922193, 0.93301270189221941], [-0.93301270189221941, 0.93301270189221941], [-0.96592582628906831, 0.96592582628906831]] keywords = { 'wrapped': [False, True], 'angular_units': [OcgisUnits.DEGREES, OcgisUnits.RADIANS], 'other_crs': [Cartesian(), WGS84()] } for k in self.iter_product_keywords(keywords): spherical = Spherical(angular_units=k.angular_units) tp = Tripole(spherical=spherical) grid = create_gridxy_global(resolution=30.0, wrapped=k.wrapped) if not k.wrapped: x_value = grid.x.get_value() select = x_value > 180. x_value[select] -= 360. grid.expand() x = grid.x.get_value() y = grid.y.get_value() if k.angular_units == OcgisUnits.RADIANS: x *= ConversionFactor.DEG_TO_RAD y *= ConversionFactor.DEG_TO_RAD z = np.ones(x.shape, dtype=x.dtype) desired = (x, y, z) try: as_cart = tp.transform_coordinates(k.other_crs, x, y, z) except CRSNotEquivalenError: self.assertNotEqual(k.other_crs, Cartesian()) continue x_cart, y_cart, z_cart = as_cart for idx, ii in enumerate(as_cart): actual_min_max = [ii.min(), ii.max()] self.assertNumpyAllClose(np.array(actual_min_max), np.array(desired_min_maxes[idx])) actual = tp.transform_coordinates(k.other_crs, x_cart, y_cart, z_cart, inverse=True) for a, d in zip(actual, desired): are = np.abs(a - d) self.assertLessEqual(are.max(), 1e-6)
def test_system_geometry_identifier_typed_appropriately(self): """Test GID is typed appropriately according to the data model.""" ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'} grid = create_gridxy_global(resolution=3.0) field = create_exact_field(grid, 'foo', crs=Spherical()) ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True) actual = ops.execute() actual = actual.get_element(container_ugid=1) self.assertEqual(actual.geom.ugid.dtype, np.int32) # Test data model is retrieved appropriately from file. ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'} grid = create_gridxy_global(resolution=3.0) field = create_exact_field(grid, 'foo', crs=Spherical()) ops = OcgOperations(dataset=field, output_format_options=ofo, output_format='nc') ret = ops.execute() rd = RequestDataset(uri=ret) ops = OcgOperations(dataset=rd, geom=[-100, 30, -90, 40], aggregate=True) actual = ops.execute() actual = actual.get_element(container_ugid=1) self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_create_merged_weight_file(self): import ESMF path_src = self.get_temporary_file_path('src.nc') path_dst = self.get_temporary_file_path('dst.nc') src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical()) dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical()) src_grid.write(path_src) dst_grid.write(path_dst) # Split source and destination grids --------------------------------------------------------------------------- gs = GridChunker(src_grid, dst_grid, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths, genweights=True) gs.write_chunks() # Merge weight files ------------------------------------------------------------------------------------------- merged_weight_filename = self.get_temporary_file_path('merged_weights.nc') gs.create_merged_weight_file(merged_weight_filename) # Generate a global weight file using ESMF --------------------------------------------------------------------- global_weights_filename = self.get_temporary_file_path('global_weights.nc') srcgrid = ESMF.Grid(filename=path_src, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) dstgrid = ESMF.Grid(filename=path_dst, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) srcfield = ESMF.Field(grid=srcgrid) dstfield = ESMF.Field(grid=dstgrid) _ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=global_weights_filename, regrid_method=ESMF.RegridMethod.CONSERVE) # Test merged and global weight files are equivalent ----------------------------------------------------------- self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_chunked_rwg_spatial_subset(self): env.CLOBBER_UNITS_ON_BOUNDS = False src_grid = create_gridxy_global(crs=Spherical()) src_field = create_exact_field(src_grid, 'foo') xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim') yvar = Variable(name='y', value=[40., 50.], dimensions='ydim') dst_grid = Grid(x=xvar, y=yvar, crs=Spherical()) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_grid.parent.write(destination) wd = os.path.join(self.current_dir_output, 'chunks') weight = os.path.join(self.current_dir_output, 'weights.nc') spatial_subset = os.path.join(self.current_dir_output, 'spatial_subset.nc') runner = CliRunner() cli_args = [ 'chunked-rwg', '--source', source, '--destination', destination, '--wd', wd, '--spatial_subset', '--spatial_subset_path', spatial_subset, '--weight', weight, '--esmf_regrid_method', 'BILINEAR', '--persist' ] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) actual = RequestDataset(uri=spatial_subset).create_field() actual_ymean = actual.grid.get_value_stacked()[0].mean() actual_xmean = actual.grid.get_value_stacked()[1].mean() self.assertEqual(actual_ymean, 45.) self.assertEqual(actual_xmean, -85.) self.assertEqual(actual.grid.shape, (14, 14)) self.assertTrue(os.path.exists(weight)) actual = RequestDataset(weight, driver='netcdf').create_field() self.assertIn('history', actual.attrs)
def test_get_dimension_map_2d_spatial_coordinates(self): grid = create_gridxy_global() grid.expand() path = self.get_temporary_file_path('foo.nc') f = Field(grid=grid) f.write(path) rd = RequestDataset(path) field = rd.get() sub = field.get_field_slice({'y': 10, 'x': 5}) self.assertEqual(sub.grid.x.shape, (1, 1)) actual = f.dimension_map.get_dimension(DimensionMapKey.Y) self.assertEqual(actual, ['y']) actual = f.dimension_map.get_dimension(DimensionMapKey.X) self.assertEqual(actual, ['x'])
def test_convert_to_geometry_coordinates_polygons(self): grid = create_gridxy_global(resolution=45.0) geom = grid.get_abstraction_geometry() geom.reshape(Dimension('n_elements', geom.size)) keywords = dict(pack=[True, False], repeat_last_node=[False, True], start_index=[0, 1]) for k in self.iter_product_keywords(keywords): actual = geom.convert_to(pack=k.pack, repeat_last_node=k.repeat_last_node, start_index=k.start_index) self.assertEqual(actual.cindex.attrs['start_index'], k.start_index) self.assertEqual(actual.start_index, k.start_index) self.assertEqual(actual.cindex.get_value()[0].min(), k.start_index) self.assertEqual(actual.packed, k.pack) self.assertIsNotNone(actual.cindex) for actual_geom, desired_geom in zip(actual.get_geometry_iterable(), geom.get_value().flat): self.assertEqual(actual_geom[1], desired_geom)
def test_create_dimension_map_2d_spatial_coordinates(self): grid = create_gridxy_global() grid.expand() path = self.get_temporary_file_path('foo.nc') f = Field(grid=grid) f.write(path) rd = RequestDataset(path) field = rd.get() sub = field.get_field_slice({'y': 10, 'x': 5}) self.assertEqual(sub.grid.x.shape, (1, 1)) actual = f.dimension_map.get_dimension(DimensionMapKey.Y) self.assertEqual(actual, ['y']) actual = f.dimension_map.get_dimension(DimensionMapKey.X) self.assertEqual(actual, ['x'])
def test_system_merge_geometries_across_shapefiles(self): geoms_to_union = [] state_names = ('Nebraska', 'South Dakota', 'North Dakota') gci = GeomCabinetIterator(path=self.path_state_boundaries) for row in gci: if row['properties']['STATE_NAME'] in state_names: geoms_to_union.append(row['geom']) self.assertEqual(len(geoms_to_union), 3) unioned = cascaded_union(geoms_to_union) grid = create_gridxy_global() field = create_exact_field(grid, 'data', crs=WGS84()) original_shape = field.grid.shape ops = OcgOperations(dataset=field, geom=unioned) ret = ops.execute() actual_shape = ret.get_element().grid.shape self.assertNotEqual(actual_shape, original_shape)
def test_system_masking_with_smm(self): """Test masking with sparse matrix multiplication.""" from ocgis.regrid import RegridOperation grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0) src_field = create_exact_field(grid, 'exact', ntime=3) mask = src_field.grid.get_mask(create=True) mask[0:2, :] = True mask[:, -2:] = True mask[-2:, :] = True mask[:, 0:2] = True src_field.grid.set_mask(mask, cascade=True) src_field['exact'].set_value(src_field['exact'].mv().filled()) dst_field = deepcopy(src_field) dst_field.remove_variable('exact') weights = self.get_temporary_file_path('weights.nc', collective=True) weights = vm.bcast(weights) ro = RegridOperation(src_field, dst_field, regrid_options={ 'weights_out': weights, 'split': False }) _ = ro.execute() ro2 = RegridOperation(src_field, dst_field, regrid_options={ 'weights_in': weights, 'split': True }) result = ro2.execute() actual = result['exact'].mv() desired = src_field['exact'].mv() self.assertNumpyAllClose(actual, desired)
def test_chunked_rwg_spatial_subset(self): env.CLOBBER_UNITS_ON_BOUNDS = False src_grid = create_gridxy_global(crs=Spherical()) src_field = create_exact_field(src_grid, 'foo') xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim') yvar = Variable(name='y', value=[40., 50.], dimensions='ydim') dst_grid = Grid(x=xvar, y=yvar, crs=Spherical()) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_grid.parent.write(destination) wd = os.path.join(self.current_dir_output, 'chunks') weight = os.path.join(self.current_dir_output, 'weights.nc') runner = CliRunner() cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--wd', wd, '--spatial_subset', '--weight', weight, '--esmf_regrid_method', 'BILINEAR', '--persist'] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) dst_path = os.path.join(wd, 'spatial_subset.nc') self.assertTrue(os.path.exists(weight)) actual = RequestDataset(uri=dst_path).create_field() actual_ymean = actual.grid.get_value_stacked()[0].mean() actual_xmean = actual.grid.get_value_stacked()[1].mean() self.assertEqual(actual_ymean, 45.) self.assertEqual(actual_xmean, -85.) self.assertEqual(actual.grid.shape, (14, 14))
def test_get_dimension_map_with_spatial_mask(self): path = self.get_temporary_file_path('foo.nc') grid = create_gridxy_global() gmask = grid.get_mask(create=True) gmask[1, 1] = True grid.set_mask(gmask) grid.parent.write(path) rd = RequestDataset(path) driver = DriverNetcdfCF(rd) dmap = driver.get_dimension_map(driver.metadata_source) self.assertIsNotNone(dmap.get_spatial_mask()) field = rd.get() self.assertEqual(field.grid.get_mask().sum(), 1) # Test mask variable is blown away if set to None during a read. rd = RequestDataset(path) rd.dimension_map.set_spatial_mask(None) self.assertIsNone(rd.dimension_map.get_spatial_mask()) # rd.dimension_map.pprint() field = rd.get() self.assertIsNone(field.grid.get_mask())
def test_create_dimension_map_with_spatial_mask(self): path = self.get_temporary_file_path('foo.nc') grid = create_gridxy_global() gmask = grid.get_mask(create=True) gmask[1, 1] = True grid.set_mask(gmask) grid.parent.write(path) rd = RequestDataset(path) driver = DriverNetcdfCF(rd) dmap = driver.create_dimension_map(driver.metadata_source) self.assertIsNotNone(dmap.get_spatial_mask()) field = rd.get() self.assertEqual(field.grid.get_mask().sum(), 1) # Test mask variable is blown away if set to None during a read. rd = RequestDataset(path) rd.dimension_map.set_spatial_mask(None) self.assertIsNone(rd.dimension_map.get_spatial_mask()) # rd.dimension_map.pprint() field = rd.get() self.assertIsNone(field.grid.get_mask())
def test_update_crs_to_cartesian(self): """Test a spherical to cartesian CRS update.""" grid = create_gridxy_global(resolution=30.0, wrapped=True, crs=Spherical()) z_value = np.ones(grid.shape, dtype=grid.dtype) zvar = Variable('ocgis_zc', value=z_value, dimensions=grid.dimensions) grid.z = zvar z_bounds_value = np.ones((list(zvar.shape) + [4]), dtype=grid.dtype) zvar_bounds = Variable('ocgis_zc_bounds', z_bounds_value, list(grid.dimensions) + ['corners']) grid.z.set_bounds(zvar_bounds) original_grid = deepcopy(grid) original_grid.expand() grid.update_crs(Cartesian()) self.assertIsInstance(grid.crs, Cartesian) self.assertNotAlmostEquals(original_grid.x.get_value().max(), grid.x.get_value().max()) self.assertNotAlmostEquals(original_grid.y.get_value().max(), grid.y.get_value().max()) self.assertNotAlmostEquals(original_grid.x.bounds.get_value().max(), grid.x.bounds.get_value().max()) self.assertNotAlmostEquals(original_grid.y.bounds.get_value().max(), grid.y.bounds.get_value().max()) grid.update_crs(Spherical()) self.assertEqual(grid.crs, Spherical()) self.assertNumpyAllClose(original_grid.x.get_value(), grid.x.get_value()) self.assertNumpyAllClose(original_grid.y.get_value(), grid.y.get_value()) self.assertNumpyAllClose(original_grid.x.bounds.get_value(), grid.x.bounds.get_value()) self.assertNumpyAllClose(original_grid.y.bounds.get_value(), grid.y.bounds.get_value()) self.assertEqual(grid.z.get_value().sum(), 72)
def test_convert_to_geometry_coordinates_polygons(self): grid = create_gridxy_global(resolution=45.0) geom = grid.get_abstraction_geometry() geom.reshape(Dimension('n_elements', geom.size)) keywords = dict(pack=[True, False], repeat_last_node=[False, True], start_index=[0, 1]) for k in self.iter_product_keywords(keywords): actual = geom.convert_to(pack=k.pack, repeat_last_node=k.repeat_last_node, start_index=k.start_index) self.assertEqual(actual.cindex.attrs['start_index'], k.start_index) self.assertEqual(actual.start_index, k.start_index) self.assertEqual(actual.cindex.get_value()[0].min(), k.start_index) self.assertEqual(actual.packed, k.pack) self.assertIsNotNone(actual.cindex) for actual_geom, desired_geom in zip( actual.get_geometry_iterable(), geom.get_value().flat): self.assertEqual(actual_geom[1], desired_geom)
import ocgis from ocgis.test.base import create_gridxy_global, create_exact_field # Name of the variable to subset. VAR_TAS = 'tas' # Make it easy to switch to non-snippet requests. SNIPPET = True # Set output directory for shapefile and keyed formats. (MAKE SURE IT EXISTS!) ocgis.env.DIR_OUTPUT = tempfile.mkdtemp() print ocgis.env.DIR_OUTPUT # The bounding box coordinates [minx, miny, maxx, maxy] for the state of Colorado in WGS84 latitude/longitude # coordinates. BBOX = [-109.1, 36.9, -102.0, 41.0] # Create synthetic data for this example. grid = create_gridxy_global(resolution=5.0) field = create_exact_field(grid, VAR_TAS, ntime=31) data_path = os.path.join(ocgis.env.DIR_OUTPUT, 'ocgis_example_simple_subset.nc') field.write(data_path) # This object will be reused so just build it once. Variable names are typically auto-discovered. rd = ocgis.RequestDataset(data_path, VAR_TAS) ######################################################################################################################## # Returning an OCGIS spatial collection ret = ocgis.OcgOperations(dataset=rd, geom=BBOX, snippet=SNIPPET).execute() ######################################################################################################################## # Returning conversions
import time import ocgis from ocgis.test.base import create_gridxy_global, create_exact_field ocgis.env.ADD_OPS_MPI_BARRIER = False ocgis.env.OVERWRITE = True # Path to the output netCDf file. PATH = 'foo.nc' tic = time.time() # Create a test grid. grid = create_gridxy_global() # Create an exact field on the grid. field = create_exact_field(grid, 'foo') # Write the field to disk. field.write(PATH) rd = ocgis.RequestDataset(PATH) # Calculate a monthly mean. ops = ocgis.OcgOperations(dataset=rd, calc=[{ 'func': 'mean', 'name': 'mean' }], calc_grouping=['month'], prefix='mean', dir_output='.', output_format='nc')
def test_system_spatial_averaging_through_operations_state_boundaries(self): if MPI_SIZE != 8: raise SkipTest('MPI_SIZE != 8') ntime = 3 # Get the exact field value for the state's representative center. with vm.scoped([0]): if MPI_RANK == 0: states = RequestDataset(self.path_state_boundaries, driver='vector').get() states.update_crs(env.DEFAULT_COORDSYS) fill = np.zeros((states.geom.shape[0], 2)) for idx, geom in enumerate(states.geom.get_value().flat): centroid = geom.centroid fill[idx, :] = centroid.x, centroid.y exact_states = create_exact_field_value(fill[:, 0], fill[:, 1]) state_ugid = states['UGID'].get_value() area = states.geom.area keywords = { 'spatial_operation': [ 'clip', 'intersects' ], 'aggregate': [ True, False ], 'wrapped': [True, False], 'output_format': [ OutputFormatName.OCGIS, 'csv', 'csv-shp', 'shp' ], } # total_iterations = len(list(self.iter_product_keywords(keywords))) for ctr, k in enumerate(self.iter_product_keywords(keywords)): # barrier_print(k) # if ctr % 1 == 0: # if vm.is_root: # print('Iteration {} of {}...'.format(ctr + 1, total_iterations)) with vm.scoped([0]): if vm.is_root: grid = create_gridxy_global(resolution=1.0, dist=False, wrapped=k.wrapped) field = create_exact_field(grid, 'foo', ntime=ntime) path = self.get_temporary_file_path('foo.nc') field.write(path) else: path = None path = MPI_COMM.bcast(path) rd = RequestDataset(path) ops = OcgOperations(dataset=rd, geom='state_boundaries', spatial_operation=k.spatial_operation, aggregate=k.aggregate, output_format=k.output_format, prefix=str(ctr), # geom_select_uid=[8] ) ret = ops.execute() # Test area is preserved for a problem element during union. The union's geometry was not fully represented # in the output. if k.output_format == 'shp' and k.aggregate and k.spatial_operation == 'clip': with vm.scoped([0]): if vm.is_root: inn = RequestDataset(ret).get() inn_ugid_idx = np.where(inn['UGID'].get_value() == 8)[0][0] ugid_idx = np.where(state_ugid == 8)[0][0] self.assertAlmostEqual(inn.geom.get_value()[inn_ugid_idx].area, area[ugid_idx], places=2) # Test the overview geometry shapefile is written. if k.output_format == 'shp': directory = os.path.split(ret)[0] contents = os.listdir(directory) actual = ['_ugid.shp' in c for c in contents] self.assertTrue(any(actual)) elif k.output_format == 'csv-shp': directory = os.path.split(ret)[0] directory = os.path.join(directory, 'shp') contents = os.listdir(directory) actual = ['_ugid.shp' in c for c in contents] self.assertTrue(any(actual)) if not k.aggregate: actual = ['_gid.shp' in c for c in contents] self.assertTrue(any(actual)) if k.output_format == OutputFormatName.OCGIS: geom_keys = ret.children.keys() all_geom_keys = vm.gather(np.array(geom_keys)) if vm.is_root: all_geom_keys = hgather(all_geom_keys) self.assertEqual(len(np.unique(all_geom_keys)), 51) if k.aggregate: actual = Dict() for field, container in ret.iter_fields(yield_container=True): if not field.is_empty: ugid = container.geom.ugid.get_value()[0] actual[ugid]['actual'] = field.data_variables[0].get_value() actual[ugid]['area'] = container.geom.area[0] actual = vm.gather(actual) if vm.is_root: actual = dgather(actual) ares = [] actual_areas = [] for ugid_key, v in actual.items(): ugid_idx = np.where(state_ugid == ugid_key)[0][0] desired = exact_states[ugid_idx] actual_areas.append(v['area']) for tidx in range(ntime): are = np.abs((desired + ((tidx + 1) * 10)) - v['actual'][tidx, 0]) ares.append(are) if k.spatial_operation == 'clip': diff = np.abs(np.array(area) - np.array(actual_areas)) self.assertLess(np.max(diff), 1e-6) self.assertLess(np.mean(diff), 1e-6) # Test relative errors. self.assertLess(np.max(ares), 0.031) self.assertLess(np.mean(ares), 0.009)
import os import subprocess import tempfile import ocgis from ocgis.test.base import create_gridxy_global DATADIR = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_') SRC_CFGRID = os.path.join(DATADIR, 'src.nc') DST_CFGRID = os.path.join(DATADIR, 'dst.nc') WEIGHT = os.path.join(DATADIR, 'esmf_weights.nc') # Write the source and destination grids. The destination grid has a slightly lower spatial resolution. ---------------- srcgrid = create_gridxy_global(crs=ocgis.crs.Spherical()) srcgrid.parent.write(SRC_CFGRID) dstgrid = create_gridxy_global(resolution=1.33, crs=ocgis.crs.Spherical()) dstgrid.parent.write(DST_CFGRID) # ---------------------------------------------------------------------------------------------------------------------- # Construct the chunked regrid weight generation command and execute in a subprocess. cmd = ['mpirun', '-n', str(4), 'ocli', 'chunked-rwg', '-s', SRC_CFGRID, '-d', DST_CFGRID, '-w', WEIGHT, '-n', '5,5'] print(' '.join(cmd)) # Command line looks like: # mpirun -n 4 ocli chunked-rwg -s src.nc -d dst.nc -w esmf_weights.nc -n 5,5 subprocess.check_call(cmd) # Inspect the weight file output.
import subprocess from ocgis import env from ocgis.test.base import create_gridxy_global from ocgis.variable.crs import Spherical SRC_PATH = '/home/benkoziol/htmp/rwg_test_src.nc' DST_PATH = '/home/benkoziol/htmp/rwg_test_dst.nc' WGT_PATH = '/home/benkoziol/htmp/rwg_test_weights.nc' env.CLOBBER_UNITS_ON_BOUNDS = False src_grid = create_gridxy_global(resolution=10.0, wrapped=False, crs=Spherical()) dst_grid = create_gridxy_global(resolution=20.0, wrapped=False, crs=Spherical()) src_grid.write(SRC_PATH) dst_grid.write(DST_PATH) cmd = ['ESMF_RegridWeightGen', '-s', SRC_PATH, '--src_type', 'GRIDSPEC', '-d', DST_PATH, '--dst_type', 'GRIDSPEC', '-w', WGT_PATH, '--method', 'conserve', '--weight-only'] subprocess.check_call(cmd)
from shapely.geometry import box import ocgis from ocgis.test.base import create_gridxy_global, create_exact_field assert ocgis.vm.size_global == 2 # Create synthetic data for this example. Create and write the data on a single process. Subcommunicators are always # given a name. with ocgis.vm.scoped('serial write', [0]): # There will be null communicators for the duration of the context manager. if not ocgis.vm.is_null: grid = create_gridxy_global(resolution=5.0, dist=False) field = create_exact_field(grid, 'exact', ntime=31) field.write('ocgis_parallel_example.nc') # Place a barrier so write can finish. Race conditions can occur with subcommunicators. Generally, barriers are managed # by the operations. ocgis.vm.barrier() # This is our subset geometry. bbox = [150, 30, 170, 50] bbox = box(*bbox) # Load the data from file. rd = ocgis.RequestDataset('ocgis_parallel_example.nc') field = rd.get() # By default, data is distributed along the largest spatial dimension. The x or longitude dimension in this case. distributed_dimension = field.grid.dimensions[1] bounds_local = {0: (0, 36), 1: (36, 72)}
import os import subprocess import tempfile import ocgis from ocgis.test.base import create_gridxy_global DATADIR = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_') SRC_CFGRID = os.path.join(DATADIR, 'src.nc') DST_CFGRID = os.path.join(DATADIR, 'dst.nc') WEIGHT = os.path.join(DATADIR, 'esmf_weights.nc') # Write the source and destination grids. The destination grid has a much smaller spatial extent. ---------------------- srcgrid = create_gridxy_global(crs=ocgis.crs.Spherical()) srcgrid.parent.write(SRC_CFGRID) # Write the destination grid. Slice the grid first to create a single cell. dstgrid = create_gridxy_global(resolution=5, crs=ocgis.crs.Spherical()) dstgrid = dstgrid[18, 36] dstgrid.parent.write(DST_CFGRID) # ---------------------------------------------------------------------------------------------------------------------- # Construct the chunked regrid weight generation command and execute in a subprocess. cmd = [ 'ocli', 'chunked-rwg', '-s', SRC_CFGRID, '-d', DST_CFGRID, '-w', WEIGHT, '--spatial_subset' ] print(' '.join(cmd)) # Command looks like:
def test_system_chunked_versus_global(self): """Test weight files are equivalent using the chunked versus global weight generation and SMM approach.""" if ocgis.vm.size not in [1, 4]: raise SkipTest('ocgis.vm.size not in [1, 4]') import ESMF # Do not put units on bounds variables. env.CLOBBER_UNITS_ON_BOUNDS = False # Create source and destination files. ------------------------------------------------------------------------- src_grid = create_gridxy_global(resolution=15, dist_dimname='x') dst_grid = create_gridxy_global(resolution=12, dist_dimname='x') src_field = create_exact_field(src_grid, 'foo', crs=Spherical(), dtype=np.float64) dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical(), dtype=np.float64) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_field['foo'].v()[:] = -9999 dst_field.write(destination) # -------------------------------------------------------------------------------------------------------------- # Directory for output grid chunks. wd = os.path.join(self.current_dir_output, 'chunks') # Path to the merged weight file. weight = self.get_temporary_file_path('merged_weights.nc') # Generate the source and destination chunks and a merged weight file. runner = CliRunner() cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--nchunks_dst', '2,3', '--wd', wd, '--weight', weight, '--persist'] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) self.assertTrue(len(os.listdir(wd)) > 3) # Also apply the sparse matrix runner2 = CliRunner() cli_args = ['chunked-smm', '--wd', wd, '--insert_weighted', '--destination', destination] result = runner2.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) # Create a standard ESMF weights file from the original grid files. esmf_weights_path = self.get_temporary_file_path('esmf_desired_weights.nc') # Generate weights using ESMF command line interface. # cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type', # 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log'] # subprocess.check_call(cmd) # Create a weights file using the ESMF Python interface. srcgrid = ESMF.Grid(filename=source, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) dstgrid = ESMF.Grid(filename=destination, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) srcfield = ESMF.Field(grid=srcgrid, typekind=ESMF.TypeKind.R8) srcfield.data[:] = np.swapaxes(np.squeeze(src_field['foo'].v()), 0, 1) dstfield = ESMF.Field(grid=dstgrid, typekind=ESMF.TypeKind.R8) _ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=esmf_weights_path, regrid_method=ESMF.RegridMethod.CONSERVE) if ocgis.vm.rank == 0: # Assert the weight files are equivalent using chunked versus global creation. self.assertWeightFilesEquivalent(esmf_weights_path, weight) actual_dst = RequestDataset(uri=destination, decomp_type=DecompositionType.ESMF).create_field()['foo'].v() actual_dst = np.swapaxes(np.squeeze(actual_dst), 0, 1) desired_dst = dstfield.data self.assertNumpyAllClose(actual_dst, desired_dst)
select_y1 = py >= bbox[1] select_y2 = py <= bbox[3] select_y = np.logical_and(select_y1, select_y2) select = np.logical_and(select_x, select_y) if initial is not None: select = np.logical_or(select, initial) return select if __name__ == '__main__': # ------------------------------------------------------------------------------------------------------------------ # Grid splitter implementation resolution = 1. / 111. # resolution = 1. grid = create_gridxy_global(resolution=resolution, wrapped=False, crs=ocgis.crs.Spherical()) field = create_exact_field(grid, 'exact', ntime=3, fill_data_var=False, crs=ocgis.crs.Spherical()) field.write(os.path.join(OUTDIR, 'dst_field_1km.nc')) gs = GridChunker(grid, grid, (10, 10)) ctr = 1 for grid_sub in gs.iter_dst_grid_subsets(): subset_filename = os.path.join(OUTDIR, 'src_subset_{}.nc'.format(ctr)) dst_subset_filename = os.path.join(OUTDIR, 'dst_subset_{}.nc'.format(ctr)) if vm.rank == 0: print 'creating subset:', subset_filename
def test_write_esmf_weights(self): # Create source and destination fields. This is the identity test, so the source and destination fields are # equivalent. src_grid = create_gridxy_global(resolution=3.0, crs=Spherical()) # Only test masking in serial to make indexing easier...just being lazy if vm.size == 1: mask = src_grid.get_mask(create=True) mask[4, 5] = True mask[25, 27] = True src_grid.set_mask(mask) self.assertEqual(src_grid.get_mask().sum(), 2) src_field = create_exact_field(src_grid, 'foo', ntime=3) dst_field = deepcopy(src_field) # Write the fields to disk for use in global file reconstruction and testing. if vm.rank == 0: master_path = self.get_temporary_file_path('foo.nc') src_field_path = self.get_temporary_file_path('src_field.nc') else: master_path = None src_field_path = None master_path = vm.bcast(master_path) src_field_path = vm.bcast(src_field_path) assert not os.path.exists(master_path) dst_field.write(master_path) src_field.write(src_field_path) # Remove the destination data variable to test its creation and filling dst_field.remove_variable('foo') # Chunk the fields and generate weights paths = {'wd': self.current_dir_output} gc = GridChunker(src_field, dst_field, nchunks_dst=(2, 2), genweights=True, paths=paths, esmf_kwargs={'regrid_method': 'BILINEAR'}) gc.write_chunks() # This is the path to the index file describing how to reconstruct the grid file index_path = os.path.join(self.current_dir_output, gc.paths['index_file']) # Execute the sparse matrix multiplication using weights read from file gc.smm(index_path, paths['wd']) with vm.scoped('index and reconstruct', [0]): if not vm.is_null: # Reconstruct the global destination file gc.insert_weighted(index_path, self.current_dir_output, master_path) # Load the actual values from file (destination) actual_field = RequestDataset(master_path).create_field() actual = actual_field.data_variables[0].mv() # Load the desired data from file (original values in the source field) desired = RequestDataset( src_field_path).create_field().data_variables[0].mv() if vm.size_global == 1: # Masking is only tested in serial self.assertEqual(actual_field.grid.get_mask().sum(), 2) else: self.assertIsNone(actual_field.grid.get_mask()) self.assertNumpyAll(actual, desired)
def test_system_negative_values_in_spherical_grid(self): original_dir = os.getcwd() try: xcn = np.arange(-10, 350, step=10, dtype=float) xc = np.arange(0, 360, step=10, dtype=float) yc = np.arange(-90, 100, step=10, dtype=float) xvn = Variable("lon", xcn, dimensions=["lon"]) xv = Variable("lon", xc, dimensions=["lon"]) yv = Variable("lat", yc, dimensions=["lat"]) gridn = Grid(x=xvn.copy(), y=yv.copy(), crs=Spherical()) gridu = Grid(x=xv.copy(), y=yv.copy(), crs=Spherical()) gridw = create_gridxy_global(5, with_bounds=False, crs=Spherical()) grids = [gridn, gridu, gridw] for ctr, (src, dst) in enumerate(itertools.product(grids, grids)): os.chdir(self.current_dir_output) gdirname = "grid-ctr-{}".format(ctr) self.dprint(gdirname) griddir = os.path.join(self.current_dir_output, gdirname) os.mkdir(gdirname) os.chdir(gdirname) srcgridname = "gridn.nc" src.parent.write(srcgridname) dstgridname = "grid.nc" dst.parent.write(dstgridname) nchunks_dst = [(4, 1), (3, 1), (2, 1), (1, 1)] for ctr, n in enumerate(nchunks_dst): os.chdir(griddir) dirname = 'ctr-{}'.format(ctr) os.mkdir(dirname) os.chdir(dirname) wd = os.getcwd() self.dprint("current chunks", n) g = GridChunker(src, dst, nchunks_dst=n, genweights=True, paths={'wd': wd}, esmf_kwargs={'regrid_method': 'BILINEAR'}) if not g.is_one_chunk: g.write_chunks() g.create_merged_weight_file( os.path.join(griddir, "ctr-{}".format(ctr), "merged-weights.nc")) else: g.write_esmf_weights( os.path.join(griddir, srcgridname), os.path.join(griddir, dstgridname), os.path.join(griddir, "global-weights.nc")) os.chdir(griddir) for ctr in range(0, len(nchunks_dst) - 1): src_filename = os.path.join(griddir, "ctr-{}".format(ctr), "merged-weights.nc") dst_filename = os.path.join(griddir, "global-weights.nc") self.assertWeightFilesEquivalent(src_filename, dst_filename) finally: os.chdir(original_dir)
def run_create_merged_weight_file(self, filemode): import ESMF esmf_filemode = getattr(ESMF.FileMode, filemode) path_src = self.get_temporary_file_path('src.nc') path_dst = self.get_temporary_file_path('dst.nc') src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical()) dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical()) src_grid.write(path_src) dst_grid.write(path_dst) # Split source and destination grids --------------------------------------------------------------------------- src_rd = RequestDataset(path_src, driver='netcdf-cf') dst_rd = RequestDataset(path_dst, driver='netcdf-cf') gs = GridChunker(src_rd, dst_rd, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths, genweights=True, filemode=filemode) gs.write_chunks() if filemode == "WITHAUX": weightfile = self.get_temporary_file_path('esmf_weights_1.nc') vc = RequestDataset(weightfile, driver='netcdf').create_field() self.assertGreater(len(vc.keys()), 3) weightfile = self.get_temporary_file_path('esmf_weights_2.nc') vc = RequestDataset(weightfile, driver='netcdf').get() self.assertEqual(len(vc.keys()), 3) # Merge weight files ------------------------------------------------------------------------------------------- merged_weight_filename = self.get_temporary_file_path( 'merged_weights.nc') gs.create_merged_weight_file(merged_weight_filename) nvars = len( RequestDataset(merged_weight_filename, driver='netcdf').get().keys()) if filemode == "WITHAUX": self.assertGreater(nvars, 3) else: self.assertEqual(nvars, 3) # Generate a global weight file using ESMF --------------------------------------------------------------------- global_weights_filename = self.get_temporary_file_path( 'global_weights.nc') srcgrid = ESMF.Grid(filename=path_src, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) dstgrid = ESMF.Grid(filename=path_dst, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) srcfield = ESMF.Field(grid=srcgrid) dstfield = ESMF.Field(grid=dstgrid) _ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=global_weights_filename, regrid_method=ESMF.RegridMethod.CONSERVE, filemode=esmf_filemode, src_file=path_src, dst_file=path_dst, src_file_type=ESMF.FileFormat.GRIDSPEC, dst_file_type=ESMF.FileFormat.GRIDSPEC) # Test merged and global weight files are equivalent ----------------------------------------------------------- self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_write_esmf_weights(self): # Create source and destination fields. This is the identity test, so the source and destination fields are # equivalent. src_grid = create_gridxy_global(resolution=3.0, crs=Spherical()) # Only test masking in serial to make indexing easier...just being lazy if vm.size == 1: mask = src_grid.get_mask(create=True) mask[4, 5] = True mask[25, 27] = True src_grid.set_mask(mask) self.assertEqual(src_grid.get_mask().sum(), 2) src_field = create_exact_field(src_grid, 'foo', ntime=3) dst_field = deepcopy(src_field) # Write the fields to disk for use in global file reconstruction and testing. if vm.rank == 0: master_path = self.get_temporary_file_path('foo.nc') src_field_path = self.get_temporary_file_path('src_field.nc') else: master_path = None src_field_path = None master_path = vm.bcast(master_path) src_field_path = vm.bcast(src_field_path) assert not os.path.exists(master_path) dst_field.write(master_path) src_field.write(src_field_path) # Remove the destination data variable to test its creation and filling dst_field.remove_variable('foo') # Chunk the fields and generate weights paths = {'wd': self.current_dir_output} gc = GridChunker(src_field, dst_field, nchunks_dst=(2, 2), genweights=True, paths=paths, esmf_kwargs={'regrid_method': 'BILINEAR'}) gc.write_chunks() # This is the path to the index file describing how to reconstruct the grid file index_path = os.path.join(self.current_dir_output, gc.paths['index_file']) # Execute the sparse matrix multiplication using weights read from file gc.smm(index_path, paths['wd']) with vm.scoped('index and reconstruct', [0]): if not vm.is_null: # Reconstruct the global destination file gc.insert_weighted(index_path, self.current_dir_output, master_path) # Load the actual values from file (destination) actual_field = RequestDataset(master_path).create_field() actual = actual_field.data_variables[0].mv() # Load the desired data from file (original values in the source field) desired = RequestDataset(src_field_path).create_field().data_variables[0].mv() if vm.size_global == 1: # Masking is only tested in serial self.assertEqual(actual_field.grid.get_mask().sum(), 2) else: self.assertIsNone(actual_field.grid.get_mask()) self.assertNumpyAll(actual, desired)
def test_system_chunked_versus_global(self): """Test weight files are equivalent using the chunked versus global weight generation approach.""" if ocgis.vm.size not in [1, 4]: raise SkipTest('ocgis.vm.size not in [1, 4]') import ESMF # Do not put units on bounds variables. env.CLOBBER_UNITS_ON_BOUNDS = False # Create source and destination files. ------------------------------------------------------------------------- src_grid = create_gridxy_global(resolution=15) dst_grid = create_gridxy_global(resolution=12) src_field = create_exact_field(src_grid, 'foo', crs=Spherical()) dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical()) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_field.write(destination) # -------------------------------------------------------------------------------------------------------------- # Directory for output grid chunks. wd = os.path.join(self.current_dir_output, 'chunks') # Path to the merged weight file. weight = self.get_temporary_file_path('merged_weights.nc') # Generate the source and destination chunks and a merged weight file. runner = CliRunner() cli_args = [ 'chunked_rwg', '--source', source, '--destination', destination, '--nchunks_dst', '2,3', '--wd', wd, '--weight', weight, '--persist' ] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) self.assertTrue(len(os.listdir(wd)) > 3) # Create a standard ESMF weights file from the original grid files. esmf_weights_path = self.get_temporary_file_path( 'esmf_desired_weights.nc') # Generate weights using ESMF command line interface. # cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type', # 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log'] # subprocess.check_call(cmd) # Create a weights file using the ESMF Python interface. srcgrid = ESMF.Grid(filename=source, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) dstgrid = ESMF.Grid(filename=destination, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) srcfield = ESMF.Field(grid=srcgrid) dstfield = ESMF.Field(grid=dstgrid) _ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=esmf_weights_path, regrid_method=ESMF.RegridMethod.CONSERVE) if ocgis.vm.rank == 0: # Assert the weight files are equivalent using chunked versus global creation. self.assertWeightFilesEquivalent(esmf_weights_path, weight)
""" Provides a general overview of the OCGIS spatial collection object. """ from ocgis import SpatialCollection, GeometryVariable, Variable, crs from ocgis.test.base import create_gridxy_global, create_exact_field from ocgis.util.helpers import pprint_dict from shapely.geometry import box # Spatial collections are hierarchical and may contain groups (children). In OCGIS, the first spatial collection groups # are subset geometries. The children or groups within the subset geometry groups contain the data subset by the # geometry. This is easiest to understand by walking through how OCGIS subsets and assembles a spatial collection inside # operations. # Create a test grid. grid = create_gridxy_global() # Create an exact field on the grid. field = create_exact_field(grid, 'foo') # Create a subset geometry. subset_geom = box(30., 20., 40., 25.) # Subset the field using the geometry. Note we subset the grid and return its parent (the field). sub = field.grid.get_intersects(subset_geom).parent # Subset geometries are themselves treated as fields. Convert the geometry to an OCGIS geometry variable. gvar = GeometryVariable.from_shapely(subset_geom, ugid=11, crs=crs.Spherical()) # Add some descriptive variables. info = Variable(name='desc', value=['random bbox'], dtype=str, dimensions=gvar.dimensions[0])
import subprocess from ocgis import env from ocgis.test.base import create_gridxy_global from ocgis.variable.crs import Spherical SRC_PATH = '/home/benkoziol/htmp/rwg_test_src.nc' DST_PATH = '/home/benkoziol/htmp/rwg_test_dst.nc' WGT_PATH = '/home/benkoziol/htmp/rwg_test_weights.nc' env.CLOBBER_UNITS_ON_BOUNDS = False src_grid = create_gridxy_global(resolution=10.0, wrapped=False, crs=Spherical()) dst_grid = create_gridxy_global(resolution=20.0, wrapped=False, crs=Spherical()) src_grid.write(SRC_PATH) dst_grid.write(DST_PATH) cmd = [ 'ESMF_RegridWeightGen', '-s', SRC_PATH, '--src_type', 'GRIDSPEC', '-d', DST_PATH, '--dst_type', 'GRIDSPEC', '-w', WGT_PATH, '--method', 'conserve', '--weight-only' ] subprocess.check_call(cmd)
from ocgis import RequestDataset from ocgis.test.base import create_gridxy_global, create_exact_field from ocgis.variable.crs import Spherical # Create synthetic data for this example. grid = create_gridxy_global(resolution=5.0) field = create_exact_field(grid, 'exact', ntime=31, crs=Spherical()) field.write('ocgis_example_inspect_request_dataset.nc') # Create the request dataset object. rd = RequestDataset('ocgis_example_inspect_request_dataset.nc') # Provides a metadata dump for the request dataset. rd.inspect() # These are the auto-discovered data variable names. assert rd.variable == 'exact' # The dimension map provides information on how OCGIS will interpret the dataset. rd.dimension_map.pprint()