def test_get_ocgis_field_from_esmf_field(self): from ocgis.regrid.base import get_esmf_field_from_ocgis_field from ocgis.regrid.base import get_ocgis_field_from_esmf_field ogrid = create_gridxy_global(crs=Spherical()) ofield = create_exact_field(ogrid, 'foo', ntime=3) ogrid = ofield.grid ogrid_mask = ogrid.get_mask(create=True) ogrid_mask[1, 0] = True ogrid.set_mask(ogrid_mask) efield = get_esmf_field_from_ocgis_field(ofield) self.assertEqual(efield.data.shape, (360, 180, 3)) ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield) actual_dv_mask = ofield_actual.data_variables[0].get_mask() self.assertTrue(np.all(actual_dv_mask[:, 1, 0])) self.assertEqual(actual_dv_mask.sum(), 3) self.assertNumpyAll(ofield.data_variables[0].get_value(), ofield_actual.data_variables[0].get_value()) self.assertEqual(ofield.data_variables[0].name, efield.name) self.assertNumpyAll(ofield.time.get_value(), ofield_actual.time.get_value())
def test_system_masking_with_smm(self): """Test masking with sparse matrix multiplication.""" from ocgis.regrid import RegridOperation grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0) src_field = create_exact_field(grid, 'exact', ntime=3) mask = src_field.grid.get_mask(create=True) mask[0:2, :] = True mask[:, -2:] = True mask[-2:, :] = True mask[:, 0:2] = True src_field.grid.set_mask(mask, cascade=True) src_field['exact'].set_value(src_field['exact'].mv().filled()) dst_field = deepcopy(src_field) dst_field.remove_variable('exact') weights = self.get_temporary_file_path('weights.nc', collective=True) weights = vm.bcast(weights) ro = RegridOperation(src_field, dst_field, regrid_options={'weights_out': weights, 'split': False}) _ = ro.execute() ro2 = RegridOperation(src_field, dst_field, regrid_options={'weights_in': weights, 'split': True}) result = ro2.execute() actual = result['exact'].mv() desired = src_field['exact'].mv() self.assertNumpyAllClose(actual, desired)
def test_system_multiple_netcdf_files(self): """Test subsetting multiple netCDF files and returning a spatial collection.""" grid = create_gridxy_global(resolution=3.0) vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax'] paths = [self.get_temporary_file_path('{}.nc'.format(ii)) for ii in vars] geom_select_uid = [16, 23] field_names = ['tasmin', 'tas', 'tasmax'] for ctr, (path, var) in enumerate(zip(paths, vars), start=1): field = create_exact_field(grid.copy(), var, ntime=3) field.data_variables[0].get_value()[:] = 10 * ctr field.write(path) rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in zip(paths, vars, field_names)] ops = OcgOperations(dataset=rds, spatial_operation='clip', aggregate=True, geom=self.path_state_boundaries, geom_select_uid=geom_select_uid) ret = ops.execute() self.assertAsSetEqual(ret.children.keys(), geom_select_uid) for geom_uid in geom_select_uid: actual = ret.children[geom_uid].children.keys() self.assertAsSetEqual(actual, field_names) for idx, field_name in enumerate(field_names): actual = ret.get_element(container_ugid=geom_uid, field_name=field_names[idx], variable_name=vars[idx]) actual = actual.get_value() actual = actual == (idx + 1) * 10 self.assertTrue(np.all(actual))
def test_to_xarray(self): grid = create_gridxy_global(crs=Spherical()) field = create_exact_field(grid, 'foo', ntime=3) field.attrs['i_am_global'] = 'confirm' field.grid.abstraction = Topology.POINT field.set_abstraction_geom() field.time.set_extrapolated_bounds('time_bounds', 'bounds') xr = field.to_xarray() self.assertEqual(xr.attrs['i_am_global'], 'confirm') self.assertGreater(len(xr.coords), 0)
def test_system_user_geometry_identifier_typed_appropriately(self): """Test UGID is typed appropriately according to the data model.""" ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'} grid = create_gridxy_global(resolution=3.0) field = create_exact_field(grid, 'foo', crs=Spherical()) ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True) actual = ops.execute() actual = actual[1] self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_system(self): from ocgis.regrid.base import create_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects import ESMF yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float) xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float) ogrid = Grid(y=yc, x=xc, crs=Spherical()) ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds') np.random.seed(1) mask = np.random.rand(*ogrid.shape) mask = mask > 0.5 self.assertTrue(mask.sum() > 3) ogrid.set_mask(mask) egrid = create_esmf_grid(ogrid) actual_shape = egrid.size[0].tolist() desired_shape = np.flipud(ogrid.shape).tolist() self.assertEqual(actual_shape, desired_shape) desired = ogrid.get_value_stacked() desired = np.ma.array(desired, mask=False) desired.mask[0, :, :] = ogrid.get_mask() desired.mask[1, :, :] = ogrid.get_mask() desired = desired.sum() actual_col = egrid.get_coords(0) actual_row = egrid.get_coords(1) actual_mask = np.invert(egrid.mask[0].astype(bool)) actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(actual_col, mask=actual_mask).sum() self.assertEqual(actual, desired) desired = 9900.0 corners = egrid.coords[ESMF.StaggerLoc.CORNER] actual = corners[0].sum() + corners[1].sum() self.assertEqual(actual, desired) ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical()) variable_name, efield, tidx = list(iter_esmf_fields(ofield, split=False))[0] desired_value = ofield['data'].get_value() self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3) destroy_esmf_objects([egrid, efield]) ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True) desired_value = ofield['data'].get_masked_value() keywords = dict(split=[False, True]) for k in self.iter_product_keywords(keywords): opts = {'split': k.split} dofield = ofield.deepcopy() dofield['data'].get_value().fill(0) ro = RegridOperation(ofield, dofield, regrid_options=opts) actual_field = ro.execute() actual_value = actual_field['data'].get_masked_value() self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def test_system_geometry_identifier_typed_appropriately(self): """Test GID is typed appropriately according to the data model.""" ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'} grid = create_gridxy_global(resolution=3.0) field = create_exact_field(grid, 'foo', crs=Spherical()) ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True) actual = ops.execute() actual = actual.get_element(container_ugid=1) self.assertEqual(actual.geom.ugid.dtype, np.int32) # Test data model is retrieved appropriately from file. ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'} grid = create_gridxy_global(resolution=3.0) field = create_exact_field(grid, 'foo', crs=Spherical()) ops = OcgOperations(dataset=field, output_format_options=ofo, output_format='nc') ret = ops.execute() rd = RequestDataset(uri=ret) ops = OcgOperations(dataset=rd, geom=[-100, 30, -90, 40], aggregate=True) actual = ops.execute() actual = actual.get_element(container_ugid=1) self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_chunked_rwg_spatial_subset(self): env.CLOBBER_UNITS_ON_BOUNDS = False src_grid = create_gridxy_global(crs=Spherical()) src_field = create_exact_field(src_grid, 'foo') xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim') yvar = Variable(name='y', value=[40., 50.], dimensions='ydim') dst_grid = Grid(x=xvar, y=yvar, crs=Spherical()) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_grid.parent.write(destination) wd = os.path.join(self.current_dir_output, 'chunks') weight = os.path.join(self.current_dir_output, 'weights.nc') spatial_subset = os.path.join(self.current_dir_output, 'spatial_subset.nc') runner = CliRunner() cli_args = [ 'chunked-rwg', '--source', source, '--destination', destination, '--wd', wd, '--spatial_subset', '--spatial_subset_path', spatial_subset, '--weight', weight, '--esmf_regrid_method', 'BILINEAR', '--persist' ] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) actual = RequestDataset(uri=spatial_subset).create_field() actual_ymean = actual.grid.get_value_stacked()[0].mean() actual_xmean = actual.grid.get_value_stacked()[1].mean() self.assertEqual(actual_ymean, 45.) self.assertEqual(actual_xmean, -85.) self.assertEqual(actual.grid.shape, (14, 14)) self.assertTrue(os.path.exists(weight)) actual = RequestDataset(weight, driver='netcdf').create_field() self.assertIn('history', actual.attrs)
def test_system_merge_geometries_across_shapefiles(self): geoms_to_union = [] state_names = ('Nebraska', 'South Dakota', 'North Dakota') gci = GeomCabinetIterator(path=self.path_state_boundaries) for row in gci: if row['properties']['STATE_NAME'] in state_names: geoms_to_union.append(row['geom']) self.assertEqual(len(geoms_to_union), 3) unioned = cascaded_union(geoms_to_union) grid = create_gridxy_global() field = create_exact_field(grid, 'data', crs=WGS84()) original_shape = field.grid.shape ops = OcgOperations(dataset=field, geom=unioned) ret = ops.execute() actual_shape = ret.get_element().grid.shape self.assertNotEqual(actual_shape, original_shape)
def test_system_masking_with_smm(self): """Test masking with sparse matrix multiplication.""" from ocgis.regrid import RegridOperation grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0) src_field = create_exact_field(grid, 'exact', ntime=3) mask = src_field.grid.get_mask(create=True) mask[0:2, :] = True mask[:, -2:] = True mask[-2:, :] = True mask[:, 0:2] = True src_field.grid.set_mask(mask, cascade=True) src_field['exact'].set_value(src_field['exact'].mv().filled()) dst_field = deepcopy(src_field) dst_field.remove_variable('exact') weights = self.get_temporary_file_path('weights.nc', collective=True) weights = vm.bcast(weights) ro = RegridOperation(src_field, dst_field, regrid_options={ 'weights_out': weights, 'split': False }) _ = ro.execute() ro2 = RegridOperation(src_field, dst_field, regrid_options={ 'weights_in': weights, 'split': True }) result = ro2.execute() actual = result['exact'].mv() desired = src_field['exact'].mv() self.assertNumpyAllClose(actual, desired)
def test_chunked_rwg_spatial_subset(self): env.CLOBBER_UNITS_ON_BOUNDS = False src_grid = create_gridxy_global(crs=Spherical()) src_field = create_exact_field(src_grid, 'foo') xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim') yvar = Variable(name='y', value=[40., 50.], dimensions='ydim') dst_grid = Grid(x=xvar, y=yvar, crs=Spherical()) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_grid.parent.write(destination) wd = os.path.join(self.current_dir_output, 'chunks') weight = os.path.join(self.current_dir_output, 'weights.nc') runner = CliRunner() cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--wd', wd, '--spatial_subset', '--weight', weight, '--esmf_regrid_method', 'BILINEAR', '--persist'] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) dst_path = os.path.join(wd, 'spatial_subset.nc') self.assertTrue(os.path.exists(weight)) actual = RequestDataset(uri=dst_path).create_field() actual_ymean = actual.grid.get_value_stacked()[0].mean() actual_xmean = actual.grid.get_value_stacked()[1].mean() self.assertEqual(actual_ymean, 45.) self.assertEqual(actual_xmean, -85.) self.assertEqual(actual.grid.shape, (14, 14))
def test_system(self): from ocgis.regrid.base import get_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects import ESMF yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float) xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float) ogrid = Grid(y=yc, x=xc, crs=Spherical()) ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds') np.random.seed(1) mask = np.random.rand(*ogrid.shape) mask = mask > 0.5 self.assertTrue(mask.sum() > 3) ogrid.set_mask(mask) egrid = get_esmf_grid(ogrid) actual_shape = egrid.size[0].tolist() desired_shape = np.flipud(ogrid.shape).tolist() self.assertEqual(actual_shape, desired_shape) desired = ogrid.get_value_stacked() desired = np.ma.array(desired, mask=False) desired.mask[0, :, :] = ogrid.get_mask() desired.mask[1, :, :] = ogrid.get_mask() desired = desired.sum() actual_col = egrid.get_coords(0) actual_row = egrid.get_coords(1) actual_mask = np.invert(egrid.mask[0].astype(bool)) actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array( actual_col, mask=actual_mask).sum() self.assertEqual(actual, desired) desired = 9900.0 corners = egrid.coords[ESMF.StaggerLoc.CORNER] actual = corners[0].sum() + corners[1].sum() self.assertEqual(actual, desired) ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical()) variable_name, efield, tidx = list( iter_esmf_fields(ofield, split=False))[0] desired_value = ofield['data'].get_value() self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3) destroy_esmf_objects([egrid, efield]) ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True) desired_value = ofield['data'].get_masked_value() keywords = dict(split=[False, True]) for k in self.iter_product_keywords(keywords): opts = {'split': k.split} dofield = ofield.deepcopy() dofield['data'].get_value().fill(0) ro = RegridOperation(ofield, dofield, regrid_options=opts) actual_field = ro.execute() actual_value = actual_field['data'].get_masked_value() self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def test_system_chunked_versus_global(self): """Test weight files are equivalent using the chunked versus global weight generation approach.""" if ocgis.vm.size not in [1, 4]: raise SkipTest('ocgis.vm.size not in [1, 4]') import ESMF # Do not put units on bounds variables. env.CLOBBER_UNITS_ON_BOUNDS = False # Create source and destination files. ------------------------------------------------------------------------- src_grid = create_gridxy_global(resolution=15) dst_grid = create_gridxy_global(resolution=12) src_field = create_exact_field(src_grid, 'foo', crs=Spherical()) dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical()) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_field.write(destination) # -------------------------------------------------------------------------------------------------------------- # Directory for output grid chunks. wd = os.path.join(self.current_dir_output, 'chunks') # Path to the merged weight file. weight = self.get_temporary_file_path('merged_weights.nc') # Generate the source and destination chunks and a merged weight file. runner = CliRunner() cli_args = [ 'chunked_rwg', '--source', source, '--destination', destination, '--nchunks_dst', '2,3', '--wd', wd, '--weight', weight, '--persist' ] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) self.assertTrue(len(os.listdir(wd)) > 3) # Create a standard ESMF weights file from the original grid files. esmf_weights_path = self.get_temporary_file_path( 'esmf_desired_weights.nc') # Generate weights using ESMF command line interface. # cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type', # 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log'] # subprocess.check_call(cmd) # Create a weights file using the ESMF Python interface. srcgrid = ESMF.Grid(filename=source, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) dstgrid = ESMF.Grid(filename=destination, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) srcfield = ESMF.Field(grid=srcgrid) dstfield = ESMF.Field(grid=dstgrid) _ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=esmf_weights_path, regrid_method=ESMF.RegridMethod.CONSERVE) if ocgis.vm.rank == 0: # Assert the weight files are equivalent using chunked versus global creation. self.assertWeightFilesEquivalent(esmf_weights_path, weight)
from ocgis.test.base import create_gridxy_global, create_exact_field # Name of the variable to subset. VAR_TAS = 'tas' # Make it easy to switch to non-snippet requests. SNIPPET = True # Set output directory for shapefile and keyed formats. (MAKE SURE IT EXISTS!) ocgis.env.DIR_OUTPUT = tempfile.mkdtemp() print ocgis.env.DIR_OUTPUT # The bounding box coordinates [minx, miny, maxx, maxy] for the state of Colorado in WGS84 latitude/longitude # coordinates. BBOX = [-109.1, 36.9, -102.0, 41.0] # Create synthetic data for this example. grid = create_gridxy_global(resolution=5.0) field = create_exact_field(grid, VAR_TAS, ntime=31) data_path = os.path.join(ocgis.env.DIR_OUTPUT, 'ocgis_example_simple_subset.nc') field.write(data_path) # This object will be reused so just build it once. Variable names are typically auto-discovered. rd = ocgis.RequestDataset(data_path, VAR_TAS) ######################################################################################################################## # Returning an OCGIS spatial collection ret = ocgis.OcgOperations(dataset=rd, geom=BBOX, snippet=SNIPPET).execute() ######################################################################################################################## # Returning conversions
import time import ocgis from ocgis.test.base import create_gridxy_global, create_exact_field ocgis.env.ADD_OPS_MPI_BARRIER = False ocgis.env.OVERWRITE = True # Path to the output netCDf file. PATH = 'foo.nc' tic = time.time() # Create a test grid. grid = create_gridxy_global() # Create an exact field on the grid. field = create_exact_field(grid, 'foo') # Write the field to disk. field.write(PATH) rd = ocgis.RequestDataset(PATH) # Calculate a monthly mean. ops = ocgis.OcgOperations(dataset=rd, calc=[{ 'func': 'mean', 'name': 'mean' }], calc_grouping=['month'], prefix='mean', dir_output='.', output_format='nc')
def test_system_spatial_averaging_through_operations_state_boundaries(self): if MPI_SIZE != 8: raise SkipTest('MPI_SIZE != 8') ntime = 3 # Get the exact field value for the state's representative center. with vm.scoped([0]): if MPI_RANK == 0: states = RequestDataset(self.path_state_boundaries, driver='vector').get() states.update_crs(env.DEFAULT_COORDSYS) fill = np.zeros((states.geom.shape[0], 2)) for idx, geom in enumerate(states.geom.get_value().flat): centroid = geom.centroid fill[idx, :] = centroid.x, centroid.y exact_states = create_exact_field_value(fill[:, 0], fill[:, 1]) state_ugid = states['UGID'].get_value() area = states.geom.area keywords = { 'spatial_operation': [ 'clip', 'intersects' ], 'aggregate': [ True, False ], 'wrapped': [True, False], 'output_format': [ OutputFormatName.OCGIS, 'csv', 'csv-shp', 'shp' ], } # total_iterations = len(list(self.iter_product_keywords(keywords))) for ctr, k in enumerate(self.iter_product_keywords(keywords)): # barrier_print(k) # if ctr % 1 == 0: # if vm.is_root: # print('Iteration {} of {}...'.format(ctr + 1, total_iterations)) with vm.scoped([0]): if vm.is_root: grid = create_gridxy_global(resolution=1.0, dist=False, wrapped=k.wrapped) field = create_exact_field(grid, 'foo', ntime=ntime) path = self.get_temporary_file_path('foo.nc') field.write(path) else: path = None path = MPI_COMM.bcast(path) rd = RequestDataset(path) ops = OcgOperations(dataset=rd, geom='state_boundaries', spatial_operation=k.spatial_operation, aggregate=k.aggregate, output_format=k.output_format, prefix=str(ctr), # geom_select_uid=[8] ) ret = ops.execute() # Test area is preserved for a problem element during union. The union's geometry was not fully represented # in the output. if k.output_format == 'shp' and k.aggregate and k.spatial_operation == 'clip': with vm.scoped([0]): if vm.is_root: inn = RequestDataset(ret).get() inn_ugid_idx = np.where(inn['UGID'].get_value() == 8)[0][0] ugid_idx = np.where(state_ugid == 8)[0][0] self.assertAlmostEqual(inn.geom.get_value()[inn_ugid_idx].area, area[ugid_idx], places=2) # Test the overview geometry shapefile is written. if k.output_format == 'shp': directory = os.path.split(ret)[0] contents = os.listdir(directory) actual = ['_ugid.shp' in c for c in contents] self.assertTrue(any(actual)) elif k.output_format == 'csv-shp': directory = os.path.split(ret)[0] directory = os.path.join(directory, 'shp') contents = os.listdir(directory) actual = ['_ugid.shp' in c for c in contents] self.assertTrue(any(actual)) if not k.aggregate: actual = ['_gid.shp' in c for c in contents] self.assertTrue(any(actual)) if k.output_format == OutputFormatName.OCGIS: geom_keys = ret.children.keys() all_geom_keys = vm.gather(np.array(geom_keys)) if vm.is_root: all_geom_keys = hgather(all_geom_keys) self.assertEqual(len(np.unique(all_geom_keys)), 51) if k.aggregate: actual = Dict() for field, container in ret.iter_fields(yield_container=True): if not field.is_empty: ugid = container.geom.ugid.get_value()[0] actual[ugid]['actual'] = field.data_variables[0].get_value() actual[ugid]['area'] = container.geom.area[0] actual = vm.gather(actual) if vm.is_root: actual = dgather(actual) ares = [] actual_areas = [] for ugid_key, v in actual.items(): ugid_idx = np.where(state_ugid == ugid_key)[0][0] desired = exact_states[ugid_idx] actual_areas.append(v['area']) for tidx in range(ntime): are = np.abs((desired + ((tidx + 1) * 10)) - v['actual'][tidx, 0]) ares.append(are) if k.spatial_operation == 'clip': diff = np.abs(np.array(area) - np.array(actual_areas)) self.assertLess(np.max(diff), 1e-6) self.assertLess(np.mean(diff), 1e-6) # Test relative errors. self.assertLess(np.max(ares), 0.031) self.assertLess(np.mean(ares), 0.009)
# Only return the first time slice. SNIPPET = True # Data returns will overwrite in this case. Use with caution!! env.OVERWRITE = True # This is where to find the shapfiles. ocgis.env.DIR_GEOMCABINET = os.path.join(os.getcwd(), os.path.split(ocgis.test.__file__)[0], 'bin') ######################################################################################################################## # Write example datasets for use in this example. grid = create_gridxy_global(resolution=3.0) vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax'] paths = ['{}.nc'.format(ii) for ii in vars] field_names = ['tasmin', 'tas', 'tasmax'] for idx, (path, var) in enumerate(zip(paths, vars)): field = create_exact_field(grid.copy(), var, ntime=3) field.data_variables[0].get_value()[:] = idx + 1 field.write(path) # for path in paths: # RequestDataset(path).inspect() ######################################################################################################################## # Create the request dataset objects for the file paths we'd like to subset. Variable will often be auto-discovered and # default field names will be created. We'll just pass in everything here. rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in zip(paths, vars, field_names)] ######################################################################################################################## # Return in-memory as an OCGIS collection for a single geometry.
from shapely.geometry import box import ocgis from ocgis.test.base import create_gridxy_global, create_exact_field assert ocgis.vm.size_global == 2 # Create synthetic data for this example. Create and write the data on a single process. Subcommunicators are always # given a name. with ocgis.vm.scoped('serial write', [0]): # There will be null communicators for the duration of the context manager. if not ocgis.vm.is_null: grid = create_gridxy_global(resolution=5.0, dist=False) field = create_exact_field(grid, 'exact', ntime=31) field.write('ocgis_parallel_example.nc') # Place a barrier so write can finish. Race conditions can occur with subcommunicators. Generally, barriers are managed # by the operations. ocgis.vm.barrier() # This is our subset geometry. bbox = [150, 30, 170, 50] bbox = box(*bbox) # Load the data from file. rd = ocgis.RequestDataset('ocgis_parallel_example.nc') field = rd.get() # By default, data is distributed along the largest spatial dimension. The x or longitude dimension in this case. distributed_dimension = field.grid.dimensions[1] bounds_local = {0: (0, 36), 1: (36, 72)}
def test_system_chunked_versus_global(self): """Test weight files are equivalent using the chunked versus global weight generation and SMM approach.""" if ocgis.vm.size not in [1, 4]: raise SkipTest('ocgis.vm.size not in [1, 4]') import ESMF # Do not put units on bounds variables. env.CLOBBER_UNITS_ON_BOUNDS = False # Create source and destination files. ------------------------------------------------------------------------- src_grid = create_gridxy_global(resolution=15, dist_dimname='x') dst_grid = create_gridxy_global(resolution=12, dist_dimname='x') src_field = create_exact_field(src_grid, 'foo', crs=Spherical(), dtype=np.float64) dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical(), dtype=np.float64) if ocgis.vm.rank == 0: source = self.get_temporary_file_path('source.nc') else: source = None source = ocgis.vm.bcast(source) src_field.write(source) if ocgis.vm.rank == 0: destination = self.get_temporary_file_path('destination.nc') else: destination = None destination = ocgis.vm.bcast(destination) dst_field['foo'].v()[:] = -9999 dst_field.write(destination) # -------------------------------------------------------------------------------------------------------------- # Directory for output grid chunks. wd = os.path.join(self.current_dir_output, 'chunks') # Path to the merged weight file. weight = self.get_temporary_file_path('merged_weights.nc') # Generate the source and destination chunks and a merged weight file. runner = CliRunner() cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--nchunks_dst', '2,3', '--wd', wd, '--weight', weight, '--persist'] result = runner.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) self.assertTrue(len(os.listdir(wd)) > 3) # Also apply the sparse matrix runner2 = CliRunner() cli_args = ['chunked-smm', '--wd', wd, '--insert_weighted', '--destination', destination] result = runner2.invoke(ocli, args=cli_args, catch_exceptions=False) self.assertEqual(result.exit_code, 0) # Create a standard ESMF weights file from the original grid files. esmf_weights_path = self.get_temporary_file_path('esmf_desired_weights.nc') # Generate weights using ESMF command line interface. # cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type', # 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log'] # subprocess.check_call(cmd) # Create a weights file using the ESMF Python interface. srcgrid = ESMF.Grid(filename=source, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) dstgrid = ESMF.Grid(filename=destination, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True) srcfield = ESMF.Field(grid=srcgrid, typekind=ESMF.TypeKind.R8) srcfield.data[:] = np.swapaxes(np.squeeze(src_field['foo'].v()), 0, 1) dstfield = ESMF.Field(grid=dstgrid, typekind=ESMF.TypeKind.R8) _ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=esmf_weights_path, regrid_method=ESMF.RegridMethod.CONSERVE) if ocgis.vm.rank == 0: # Assert the weight files are equivalent using chunked versus global creation. self.assertWeightFilesEquivalent(esmf_weights_path, weight) actual_dst = RequestDataset(uri=destination, decomp_type=DecompositionType.ESMF).create_field()['foo'].v() actual_dst = np.swapaxes(np.squeeze(actual_dst), 0, 1) desired_dst = dstfield.data self.assertNumpyAllClose(actual_dst, desired_dst)
select_y2 = py <= bbox[3] select_y = np.logical_and(select_y1, select_y2) select = np.logical_and(select_x, select_y) if initial is not None: select = np.logical_or(select, initial) return select if __name__ == '__main__': # ------------------------------------------------------------------------------------------------------------------ # Grid splitter implementation resolution = 1. / 111. # resolution = 1. grid = create_gridxy_global(resolution=resolution, wrapped=False, crs=ocgis.crs.Spherical()) field = create_exact_field(grid, 'exact', ntime=3, fill_data_var=False, crs=ocgis.crs.Spherical()) field.write(os.path.join(OUTDIR, 'dst_field_1km.nc')) gs = GridChunker(grid, grid, (10, 10)) ctr = 1 for grid_sub in gs.iter_dst_grid_subsets(): subset_filename = os.path.join(OUTDIR, 'src_subset_{}.nc'.format(ctr)) dst_subset_filename = os.path.join(OUTDIR, 'dst_subset_{}.nc'.format(ctr)) if vm.rank == 0: print 'creating subset:', subset_filename with vm.scoped_by_emptyable('grid subset', grid_sub):
from ocgis import RequestDataset from ocgis.test.base import create_gridxy_global, create_exact_field from ocgis.variable.crs import Spherical # Create synthetic data for this example. grid = create_gridxy_global(resolution=5.0) field = create_exact_field(grid, 'exact', ntime=31, crs=Spherical()) field.write('ocgis_example_inspect_request_dataset.nc') # Create the request dataset object. rd = RequestDataset('ocgis_example_inspect_request_dataset.nc') # Provides a metadata dump for the request dataset. rd.inspect() # These are the auto-discovered data variable names. assert rd.variable == 'exact' # The dimension map provides information on how OCGIS will interpret the dataset. rd.dimension_map.pprint()
from ocgis import SpatialCollection, GeometryVariable, Variable, crs from ocgis.test.base import create_gridxy_global, create_exact_field from ocgis.util.helpers import pprint_dict from shapely.geometry import box # Spatial collections are hierarchical and may contain groups (children). In OCGIS, the first spatial collection groups # are subset geometries. The children or groups within the subset geometry groups contain the data subset by the # geometry. This is easiest to understand by walking through how OCGIS subsets and assembles a spatial collection inside # operations. # Create a test grid. grid = create_gridxy_global() # Create an exact field on the grid. field = create_exact_field(grid, 'foo') # Create a subset geometry. subset_geom = box(30., 20., 40., 25.) # Subset the field using the geometry. Note we subset the grid and return its parent (the field). sub = field.grid.get_intersects(subset_geom).parent # Subset geometries are themselves treated as fields. Convert the geometry to an OCGIS geometry variable. gvar = GeometryVariable.from_shapely(subset_geom, ugid=11, crs=crs.Spherical()) # Add some descriptive variables. info = Variable(name='desc', value=['random bbox'], dtype=str, dimensions=gvar.dimensions[0]) gvar.parent.add_variable(info, is_data=True) height = Variable(name='height', value=[30], dimensions=gvar.dimensions[0], attrs={'made_up': 'yes'}) gvar.parent.add_variable(height, is_data=True)
from ocgis.test.base import create_gridxy_global, create_exact_field # Name of the variable to subset. VAR_TAS = 'tas' # Make it easy to switch to non-snippet requests. SNIPPET = True # Set output directory for shapefile and keyed formats. (MAKE SURE IT EXISTS!) ocgis.env.DIR_OUTPUT = tempfile.mkdtemp() print ocgis.env.DIR_OUTPUT # The bounding box coordinates [minx, miny, maxx, maxy] for the state of Colorado in WGS84 latitude/longitude # coordinates. BBOX = [-109.1, 36.9, -102.0, 41.0] # Create synthetic data for this example. grid = create_gridxy_global(resolution=5.0) field = create_exact_field(grid, VAR_TAS, ntime=31) data_path = os.path.join(ocgis.env.DIR_OUTPUT, 'ocgis_example_simple_subset.nc') field.write(data_path) # This object will be reused so just build it once. Variable names are typically auto-discovered. rd = ocgis.RequestDataset(data_path, VAR_TAS) ######################################################################################################################## # Returning an OCGIS spatial collection ret = ocgis.OcgOperations(dataset=rd, geom=BBOX, snippet=SNIPPET).execute() ######################################################################################################################## # Returning conversions output_formats = ['shp', 'csv', 'csv-shp', 'nc']