def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ogrid = create_gridxy_global(crs=Spherical())
ofield = create_exact_field(ogrid, 'foo', ntime=3)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
efield = get_esmf_field_from_ocgis_field(ofield)
self.assertEqual(efield.data.shape, (360, 180, 3))
ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 3)
self.assertNumpyAll(ofield.data_variables[0].get_value(),
ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
self.assertNumpyAll(ofield.time.get_value(),
ofield_actual.time.get_value())
def test_system_masking_with_smm(self):
"""Test masking with sparse matrix multiplication."""
from ocgis.regrid import RegridOperation
grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0)
src_field = create_exact_field(grid, 'exact', ntime=3)
mask = src_field.grid.get_mask(create=True)
mask[0:2, :] = True
mask[:, -2:] = True
mask[-2:, :] = True
mask[:, 0:2] = True
src_field.grid.set_mask(mask, cascade=True)
src_field['exact'].set_value(src_field['exact'].mv().filled())
dst_field = deepcopy(src_field)
dst_field.remove_variable('exact')
weights = self.get_temporary_file_path('weights.nc', collective=True)
weights = vm.bcast(weights)
ro = RegridOperation(src_field, dst_field, regrid_options={'weights_out': weights, 'split': False})
_ = ro.execute()
ro2 = RegridOperation(src_field, dst_field, regrid_options={'weights_in': weights, 'split': True})
result = ro2.execute()
actual = result['exact'].mv()
desired = src_field['exact'].mv()
self.assertNumpyAllClose(actual, desired)
def test_system_multiple_netcdf_files(self):
"""Test subsetting multiple netCDF files and returning a spatial collection."""
grid = create_gridxy_global(resolution=3.0)
vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax']
paths = [self.get_temporary_file_path('{}.nc'.format(ii)) for ii in vars]
geom_select_uid = [16, 23]
field_names = ['tasmin', 'tas', 'tasmax']
for ctr, (path, var) in enumerate(zip(paths, vars), start=1):
field = create_exact_field(grid.copy(), var, ntime=3)
field.data_variables[0].get_value()[:] = 10 * ctr
field.write(path)
rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in
zip(paths, vars, field_names)]
ops = OcgOperations(dataset=rds, spatial_operation='clip', aggregate=True, geom=self.path_state_boundaries,
geom_select_uid=geom_select_uid)
ret = ops.execute()
self.assertAsSetEqual(ret.children.keys(), geom_select_uid)
for geom_uid in geom_select_uid:
actual = ret.children[geom_uid].children.keys()
self.assertAsSetEqual(actual, field_names)
for idx, field_name in enumerate(field_names):
actual = ret.get_element(container_ugid=geom_uid, field_name=field_names[idx], variable_name=vars[idx])
actual = actual.get_value()
actual = actual == (idx + 1) * 10
self.assertTrue(np.all(actual))
def test_to_xarray(self):
grid = create_gridxy_global(crs=Spherical())
field = create_exact_field(grid, 'foo', ntime=3)
field.attrs['i_am_global'] = 'confirm'
field.grid.abstraction = Topology.POINT
field.set_abstraction_geom()
field.time.set_extrapolated_bounds('time_bounds', 'bounds')
xr = field.to_xarray()
self.assertEqual(xr.attrs['i_am_global'], 'confirm')
self.assertGreater(len(xr.coords), 0)
def test_system_user_geometry_identifier_typed_appropriately(self):
"""Test UGID is typed appropriately according to the data model."""
ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'}
grid = create_gridxy_global(resolution=3.0)
field = create_exact_field(grid, 'foo', crs=Spherical())
ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True)
actual = ops.execute()
actual = actual[1]
self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_system(self):
from ocgis.regrid.base import create_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects
import ESMF
yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float)
xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float)
ogrid = Grid(y=yc, x=xc, crs=Spherical())
ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
np.random.seed(1)
mask = np.random.rand(*ogrid.shape)
mask = mask > 0.5
self.assertTrue(mask.sum() > 3)
ogrid.set_mask(mask)
egrid = create_esmf_grid(ogrid)
actual_shape = egrid.size[0].tolist()
desired_shape = np.flipud(ogrid.shape).tolist()
self.assertEqual(actual_shape, desired_shape)
desired = ogrid.get_value_stacked()
desired = np.ma.array(desired, mask=False)
desired.mask[0, :, :] = ogrid.get_mask()
desired.mask[1, :, :] = ogrid.get_mask()
desired = desired.sum()
actual_col = egrid.get_coords(0)
actual_row = egrid.get_coords(1)
actual_mask = np.invert(egrid.mask[0].astype(bool))
actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(actual_col, mask=actual_mask).sum()
self.assertEqual(actual, desired)
desired = 9900.0
corners = egrid.coords[ESMF.StaggerLoc.CORNER]
actual = corners[0].sum() + corners[1].sum()
self.assertEqual(actual, desired)
ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical())
variable_name, efield, tidx = list(iter_esmf_fields(ofield, split=False))[0]
desired_value = ofield['data'].get_value()
self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3)
destroy_esmf_objects([egrid, efield])
ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True)
desired_value = ofield['data'].get_masked_value()
keywords = dict(split=[False, True])
for k in self.iter_product_keywords(keywords):
opts = {'split': k.split}
dofield = ofield.deepcopy()
dofield['data'].get_value().fill(0)
ro = RegridOperation(ofield, dofield, regrid_options=opts)
actual_field = ro.execute()
actual_value = actual_field['data'].get_masked_value()
self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def test_system_geometry_identifier_typed_appropriately(self):
"""Test GID is typed appropriately according to the data model."""
ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'}
grid = create_gridxy_global(resolution=3.0)
field = create_exact_field(grid, 'foo', crs=Spherical())
ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True)
actual = ops.execute()
actual = actual.get_element(container_ugid=1)
self.assertEqual(actual.geom.ugid.dtype, np.int32)
# Test data model is retrieved appropriately from file.
ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'}
grid = create_gridxy_global(resolution=3.0)
field = create_exact_field(grid, 'foo', crs=Spherical())
ops = OcgOperations(dataset=field, output_format_options=ofo, output_format='nc')
ret = ops.execute()
rd = RequestDataset(uri=ret)
ops = OcgOperations(dataset=rd, geom=[-100, 30, -90, 40], aggregate=True)
actual = ops.execute()
actual = actual.get_element(container_ugid=1)
self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_chunked_rwg_spatial_subset(self):
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(crs=Spherical())
src_field = create_exact_field(src_grid, 'foo')
xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim')
yvar = Variable(name='y', value=[40., 50.], dimensions='ydim')
dst_grid = Grid(x=xvar, y=yvar, crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_grid.parent.write(destination)
wd = os.path.join(self.current_dir_output, 'chunks')
weight = os.path.join(self.current_dir_output, 'weights.nc')
spatial_subset = os.path.join(self.current_dir_output,
'spatial_subset.nc')
runner = CliRunner()
cli_args = [
'chunked-rwg', '--source', source, '--destination', destination,
'--wd', wd, '--spatial_subset', '--spatial_subset_path',
spatial_subset, '--weight', weight, '--esmf_regrid_method',
'BILINEAR', '--persist'
]
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
actual = RequestDataset(uri=spatial_subset).create_field()
actual_ymean = actual.grid.get_value_stacked()[0].mean()
actual_xmean = actual.grid.get_value_stacked()[1].mean()
self.assertEqual(actual_ymean, 45.)
self.assertEqual(actual_xmean, -85.)
self.assertEqual(actual.grid.shape, (14, 14))
self.assertTrue(os.path.exists(weight))
actual = RequestDataset(weight, driver='netcdf').create_field()
self.assertIn('history', actual.attrs)
def test_system_merge_geometries_across_shapefiles(self):
geoms_to_union = []
state_names = ('Nebraska', 'South Dakota', 'North Dakota')
gci = GeomCabinetIterator(path=self.path_state_boundaries)
for row in gci:
if row['properties']['STATE_NAME'] in state_names:
geoms_to_union.append(row['geom'])
self.assertEqual(len(geoms_to_union), 3)
unioned = cascaded_union(geoms_to_union)
grid = create_gridxy_global()
field = create_exact_field(grid, 'data', crs=WGS84())
original_shape = field.grid.shape
ops = OcgOperations(dataset=field, geom=unioned)
ret = ops.execute()
actual_shape = ret.get_element().grid.shape
self.assertNotEqual(actual_shape, original_shape)
def test_system_merge_geometries_across_shapefiles(self):
geoms_to_union = []
state_names = ('Nebraska', 'South Dakota', 'North Dakota')
gci = GeomCabinetIterator(path=self.path_state_boundaries)
for row in gci:
if row['properties']['STATE_NAME'] in state_names:
geoms_to_union.append(row['geom'])
self.assertEqual(len(geoms_to_union), 3)
unioned = cascaded_union(geoms_to_union)
grid = create_gridxy_global()
field = create_exact_field(grid, 'data', crs=WGS84())
original_shape = field.grid.shape
ops = OcgOperations(dataset=field, geom=unioned)
ret = ops.execute()
actual_shape = ret.get_element().grid.shape
self.assertNotEqual(actual_shape, original_shape)
def test_system_masking_with_smm(self):
"""Test masking with sparse matrix multiplication."""
from ocgis.regrid import RegridOperation
grid = create_gridxy_global(with_bounds=False,
crs=Spherical(),
dist_dimname='x',
resolution=5.0)
src_field = create_exact_field(grid, 'exact', ntime=3)
mask = src_field.grid.get_mask(create=True)
mask[0:2, :] = True
mask[:, -2:] = True
mask[-2:, :] = True
mask[:, 0:2] = True
src_field.grid.set_mask(mask, cascade=True)
src_field['exact'].set_value(src_field['exact'].mv().filled())
dst_field = deepcopy(src_field)
dst_field.remove_variable('exact')
weights = self.get_temporary_file_path('weights.nc', collective=True)
weights = vm.bcast(weights)
ro = RegridOperation(src_field,
dst_field,
regrid_options={
'weights_out': weights,
'split': False
})
_ = ro.execute()
ro2 = RegridOperation(src_field,
dst_field,
regrid_options={
'weights_in': weights,
'split': True
})
result = ro2.execute()
actual = result['exact'].mv()
desired = src_field['exact'].mv()
self.assertNumpyAllClose(actual, desired)
def test_chunked_rwg_spatial_subset(self):
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(crs=Spherical())
src_field = create_exact_field(src_grid, 'foo')
xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim')
yvar = Variable(name='y', value=[40., 50.], dimensions='ydim')
dst_grid = Grid(x=xvar, y=yvar, crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_grid.parent.write(destination)
wd = os.path.join(self.current_dir_output, 'chunks')
weight = os.path.join(self.current_dir_output, 'weights.nc')
runner = CliRunner()
cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--wd', wd, '--spatial_subset',
'--weight', weight, '--esmf_regrid_method', 'BILINEAR', '--persist']
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
dst_path = os.path.join(wd, 'spatial_subset.nc')
self.assertTrue(os.path.exists(weight))
actual = RequestDataset(uri=dst_path).create_field()
actual_ymean = actual.grid.get_value_stacked()[0].mean()
actual_xmean = actual.grid.get_value_stacked()[1].mean()
self.assertEqual(actual_ymean, 45.)
self.assertEqual(actual_xmean, -85.)
self.assertEqual(actual.grid.shape, (14, 14))
def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ogrid = create_gridxy_global(crs=Spherical())
ofield = create_exact_field(ogrid, 'foo', ntime=3)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
efield = get_esmf_field_from_ocgis_field(ofield)
self.assertEqual(efield.data.shape, (360, 180, 3))
ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 3)
self.assertNumpyAll(ofield.data_variables[0].get_value(), ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
self.assertNumpyAll(ofield.time.get_value(), ofield_actual.time.get_value())
def test_system(self):
from ocgis.regrid.base import get_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects
import ESMF
yc = Variable(name='yc',
value=np.arange(-90 + (45 / 2.), 90, 45),
dimensions='ydim',
dtype=float)
xc = Variable(name='xc',
value=np.arange(15, 360, 30),
dimensions='xdim',
dtype=float)
ogrid = Grid(y=yc, x=xc, crs=Spherical())
ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
np.random.seed(1)
mask = np.random.rand(*ogrid.shape)
mask = mask > 0.5
self.assertTrue(mask.sum() > 3)
ogrid.set_mask(mask)
egrid = get_esmf_grid(ogrid)
actual_shape = egrid.size[0].tolist()
desired_shape = np.flipud(ogrid.shape).tolist()
self.assertEqual(actual_shape, desired_shape)
desired = ogrid.get_value_stacked()
desired = np.ma.array(desired, mask=False)
desired.mask[0, :, :] = ogrid.get_mask()
desired.mask[1, :, :] = ogrid.get_mask()
desired = desired.sum()
actual_col = egrid.get_coords(0)
actual_row = egrid.get_coords(1)
actual_mask = np.invert(egrid.mask[0].astype(bool))
actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(
actual_col, mask=actual_mask).sum()
self.assertEqual(actual, desired)
desired = 9900.0
corners = egrid.coords[ESMF.StaggerLoc.CORNER]
actual = corners[0].sum() + corners[1].sum()
self.assertEqual(actual, desired)
ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical())
variable_name, efield, tidx = list(
iter_esmf_fields(ofield, split=False))[0]
desired_value = ofield['data'].get_value()
self.assertAlmostEqual(efield.data.sum(),
desired_value.sum(),
places=3)
destroy_esmf_objects([egrid, efield])
ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True)
desired_value = ofield['data'].get_masked_value()
keywords = dict(split=[False, True])
for k in self.iter_product_keywords(keywords):
opts = {'split': k.split}
dofield = ofield.deepcopy()
dofield['data'].get_value().fill(0)
ro = RegridOperation(ofield, dofield, regrid_options=opts)
actual_field = ro.execute()
actual_value = actual_field['data'].get_masked_value()
self.assertAlmostEqual(0.0,
np.abs(desired_value - actual_value).max())
def test_system_chunked_versus_global(self):
"""Test weight files are equivalent using the chunked versus global weight generation approach."""
if ocgis.vm.size not in [1, 4]:
raise SkipTest('ocgis.vm.size not in [1, 4]')
import ESMF
# Do not put units on bounds variables.
env.CLOBBER_UNITS_ON_BOUNDS = False
# Create source and destination files. -------------------------------------------------------------------------
src_grid = create_gridxy_global(resolution=15)
dst_grid = create_gridxy_global(resolution=12)
src_field = create_exact_field(src_grid, 'foo', crs=Spherical())
dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_field.write(destination)
# --------------------------------------------------------------------------------------------------------------
# Directory for output grid chunks.
wd = os.path.join(self.current_dir_output, 'chunks')
# Path to the merged weight file.
weight = self.get_temporary_file_path('merged_weights.nc')
# Generate the source and destination chunks and a merged weight file.
runner = CliRunner()
cli_args = [
'chunked_rwg', '--source', source, '--destination', destination,
'--nchunks_dst', '2,3', '--wd', wd, '--weight', weight, '--persist'
]
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
self.assertTrue(len(os.listdir(wd)) > 3)
# Create a standard ESMF weights file from the original grid files.
esmf_weights_path = self.get_temporary_file_path(
'esmf_desired_weights.nc')
# Generate weights using ESMF command line interface.
# cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type',
# 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log']
# subprocess.check_call(cmd)
# Create a weights file using the ESMF Python interface.
srcgrid = ESMF.Grid(filename=source,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=destination,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield,
dstfield=dstfield,
filename=esmf_weights_path,
regrid_method=ESMF.RegridMethod.CONSERVE)
if ocgis.vm.rank == 0:
# Assert the weight files are equivalent using chunked versus global creation.
self.assertWeightFilesEquivalent(esmf_weights_path, weight)
from ocgis.test.base import create_gridxy_global, create_exact_field
# Name of the variable to subset.
VAR_TAS = 'tas'
# Make it easy to switch to non-snippet requests.
SNIPPET = True
# Set output directory for shapefile and keyed formats. (MAKE SURE IT EXISTS!)
ocgis.env.DIR_OUTPUT = tempfile.mkdtemp()
print ocgis.env.DIR_OUTPUT
# The bounding box coordinates [minx, miny, maxx, maxy] for the state of Colorado in WGS84 latitude/longitude
# coordinates.
BBOX = [-109.1, 36.9, -102.0, 41.0]
# Create synthetic data for this example.
grid = create_gridxy_global(resolution=5.0)
field = create_exact_field(grid, VAR_TAS, ntime=31)
data_path = os.path.join(ocgis.env.DIR_OUTPUT,
'ocgis_example_simple_subset.nc')
field.write(data_path)
# This object will be reused so just build it once. Variable names are typically auto-discovered.
rd = ocgis.RequestDataset(data_path, VAR_TAS)
########################################################################################################################
# Returning an OCGIS spatial collection
ret = ocgis.OcgOperations(dataset=rd, geom=BBOX, snippet=SNIPPET).execute()
########################################################################################################################
# Returning conversions
import time
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
ocgis.env.ADD_OPS_MPI_BARRIER = False
ocgis.env.OVERWRITE = True
# Path to the output netCDf file.
PATH = 'foo.nc'
tic = time.time()
# Create a test grid.
grid = create_gridxy_global()
# Create an exact field on the grid.
field = create_exact_field(grid, 'foo')
# Write the field to disk.
field.write(PATH)
rd = ocgis.RequestDataset(PATH)
# Calculate a monthly mean.
ops = ocgis.OcgOperations(dataset=rd,
calc=[{
'func': 'mean',
'name': 'mean'
}],
calc_grouping=['month'],
prefix='mean',
dir_output='.',
output_format='nc')
def test_system_spatial_averaging_through_operations_state_boundaries(self):
if MPI_SIZE != 8:
raise SkipTest('MPI_SIZE != 8')
ntime = 3
# Get the exact field value for the state's representative center.
with vm.scoped([0]):
if MPI_RANK == 0:
states = RequestDataset(self.path_state_boundaries, driver='vector').get()
states.update_crs(env.DEFAULT_COORDSYS)
fill = np.zeros((states.geom.shape[0], 2))
for idx, geom in enumerate(states.geom.get_value().flat):
centroid = geom.centroid
fill[idx, :] = centroid.x, centroid.y
exact_states = create_exact_field_value(fill[:, 0], fill[:, 1])
state_ugid = states['UGID'].get_value()
area = states.geom.area
keywords = {
'spatial_operation': [
'clip',
'intersects'
],
'aggregate': [
True,
False
],
'wrapped': [True, False],
'output_format': [
OutputFormatName.OCGIS,
'csv',
'csv-shp',
'shp'
],
}
# total_iterations = len(list(self.iter_product_keywords(keywords)))
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# barrier_print(k)
# if ctr % 1 == 0:
# if vm.is_root:
# print('Iteration {} of {}...'.format(ctr + 1, total_iterations))
with vm.scoped([0]):
if vm.is_root:
grid = create_gridxy_global(resolution=1.0, dist=False, wrapped=k.wrapped)
field = create_exact_field(grid, 'foo', ntime=ntime)
path = self.get_temporary_file_path('foo.nc')
field.write(path)
else:
path = None
path = MPI_COMM.bcast(path)
rd = RequestDataset(path)
ops = OcgOperations(dataset=rd, geom='state_boundaries', spatial_operation=k.spatial_operation,
aggregate=k.aggregate, output_format=k.output_format, prefix=str(ctr),
# geom_select_uid=[8]
)
ret = ops.execute()
# Test area is preserved for a problem element during union. The union's geometry was not fully represented
# in the output.
if k.output_format == 'shp' and k.aggregate and k.spatial_operation == 'clip':
with vm.scoped([0]):
if vm.is_root:
inn = RequestDataset(ret).get()
inn_ugid_idx = np.where(inn['UGID'].get_value() == 8)[0][0]
ugid_idx = np.where(state_ugid == 8)[0][0]
self.assertAlmostEqual(inn.geom.get_value()[inn_ugid_idx].area, area[ugid_idx], places=2)
# Test the overview geometry shapefile is written.
if k.output_format == 'shp':
directory = os.path.split(ret)[0]
contents = os.listdir(directory)
actual = ['_ugid.shp' in c for c in contents]
self.assertTrue(any(actual))
elif k.output_format == 'csv-shp':
directory = os.path.split(ret)[0]
directory = os.path.join(directory, 'shp')
contents = os.listdir(directory)
actual = ['_ugid.shp' in c for c in contents]
self.assertTrue(any(actual))
if not k.aggregate:
actual = ['_gid.shp' in c for c in contents]
self.assertTrue(any(actual))
if k.output_format == OutputFormatName.OCGIS:
geom_keys = ret.children.keys()
all_geom_keys = vm.gather(np.array(geom_keys))
if vm.is_root:
all_geom_keys = hgather(all_geom_keys)
self.assertEqual(len(np.unique(all_geom_keys)), 51)
if k.aggregate:
actual = Dict()
for field, container in ret.iter_fields(yield_container=True):
if not field.is_empty:
ugid = container.geom.ugid.get_value()[0]
actual[ugid]['actual'] = field.data_variables[0].get_value()
actual[ugid]['area'] = container.geom.area[0]
actual = vm.gather(actual)
if vm.is_root:
actual = dgather(actual)
ares = []
actual_areas = []
for ugid_key, v in actual.items():
ugid_idx = np.where(state_ugid == ugid_key)[0][0]
desired = exact_states[ugid_idx]
actual_areas.append(v['area'])
for tidx in range(ntime):
are = np.abs((desired + ((tidx + 1) * 10)) - v['actual'][tidx, 0])
ares.append(are)
if k.spatial_operation == 'clip':
diff = np.abs(np.array(area) - np.array(actual_areas))
self.assertLess(np.max(diff), 1e-6)
self.assertLess(np.mean(diff), 1e-6)
# Test relative errors.
self.assertLess(np.max(ares), 0.031)
self.assertLess(np.mean(ares), 0.009)
# Only return the first time slice.
SNIPPET = True
# Data returns will overwrite in this case. Use with caution!!
env.OVERWRITE = True
# This is where to find the shapfiles.
ocgis.env.DIR_GEOMCABINET = os.path.join(os.getcwd(), os.path.split(ocgis.test.__file__)[0], 'bin')
########################################################################################################################
# Write example datasets for use in this example.
grid = create_gridxy_global(resolution=3.0)
vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax']
paths = ['{}.nc'.format(ii) for ii in vars]
field_names = ['tasmin', 'tas', 'tasmax']
for idx, (path, var) in enumerate(zip(paths, vars)):
field = create_exact_field(grid.copy(), var, ntime=3)
field.data_variables[0].get_value()[:] = idx + 1
field.write(path)
# for path in paths:
# RequestDataset(path).inspect()
########################################################################################################################
# Create the request dataset objects for the file paths we'd like to subset. Variable will often be auto-discovered and
# default field names will be created. We'll just pass in everything here.
rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in
zip(paths, vars, field_names)]
########################################################################################################################
# Return in-memory as an OCGIS collection for a single geometry.
from shapely.geometry import box
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
assert ocgis.vm.size_global == 2
# Create synthetic data for this example. Create and write the data on a single process. Subcommunicators are always
# given a name.
with ocgis.vm.scoped('serial write', [0]):
# There will be null communicators for the duration of the context manager.
if not ocgis.vm.is_null:
grid = create_gridxy_global(resolution=5.0, dist=False)
field = create_exact_field(grid, 'exact', ntime=31)
field.write('ocgis_parallel_example.nc')
# Place a barrier so write can finish. Race conditions can occur with subcommunicators. Generally, barriers are managed
# by the operations.
ocgis.vm.barrier()
# This is our subset geometry.
bbox = [150, 30, 170, 50]
bbox = box(*bbox)
# Load the data from file.
rd = ocgis.RequestDataset('ocgis_parallel_example.nc')
field = rd.get()
# By default, data is distributed along the largest spatial dimension. The x or longitude dimension in this case.
distributed_dimension = field.grid.dimensions[1]
bounds_local = {0: (0, 36), 1: (36, 72)}
def test_system_chunked_versus_global(self):
"""Test weight files are equivalent using the chunked versus global weight generation and SMM approach."""
if ocgis.vm.size not in [1, 4]:
raise SkipTest('ocgis.vm.size not in [1, 4]')
import ESMF
# Do not put units on bounds variables.
env.CLOBBER_UNITS_ON_BOUNDS = False
# Create source and destination files. -------------------------------------------------------------------------
src_grid = create_gridxy_global(resolution=15, dist_dimname='x')
dst_grid = create_gridxy_global(resolution=12, dist_dimname='x')
src_field = create_exact_field(src_grid, 'foo', crs=Spherical(), dtype=np.float64)
dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical(), dtype=np.float64)
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_field['foo'].v()[:] = -9999
dst_field.write(destination)
# --------------------------------------------------------------------------------------------------------------
# Directory for output grid chunks.
wd = os.path.join(self.current_dir_output, 'chunks')
# Path to the merged weight file.
weight = self.get_temporary_file_path('merged_weights.nc')
# Generate the source and destination chunks and a merged weight file.
runner = CliRunner()
cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--nchunks_dst', '2,3', '--wd',
wd, '--weight', weight, '--persist']
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
self.assertTrue(len(os.listdir(wd)) > 3)
# Also apply the sparse matrix
runner2 = CliRunner()
cli_args = ['chunked-smm', '--wd', wd, '--insert_weighted', '--destination', destination]
result = runner2.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
# Create a standard ESMF weights file from the original grid files.
esmf_weights_path = self.get_temporary_file_path('esmf_desired_weights.nc')
# Generate weights using ESMF command line interface.
# cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type',
# 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log']
# subprocess.check_call(cmd)
# Create a weights file using the ESMF Python interface.
srcgrid = ESMF.Grid(filename=source, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=destination, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid, typekind=ESMF.TypeKind.R8)
srcfield.data[:] = np.swapaxes(np.squeeze(src_field['foo'].v()), 0, 1)
dstfield = ESMF.Field(grid=dstgrid, typekind=ESMF.TypeKind.R8)
_ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=esmf_weights_path,
regrid_method=ESMF.RegridMethod.CONSERVE)
if ocgis.vm.rank == 0:
# Assert the weight files are equivalent using chunked versus global creation.
self.assertWeightFilesEquivalent(esmf_weights_path, weight)
actual_dst = RequestDataset(uri=destination, decomp_type=DecompositionType.ESMF).create_field()['foo'].v()
actual_dst = np.swapaxes(np.squeeze(actual_dst), 0, 1)
desired_dst = dstfield.data
self.assertNumpyAllClose(actual_dst, desired_dst)
select_y2 = py <= bbox[3]
select_y = np.logical_and(select_y1, select_y2)
select = np.logical_and(select_x, select_y)
if initial is not None:
select = np.logical_or(select, initial)
return select
if __name__ == '__main__':
# ------------------------------------------------------------------------------------------------------------------
# Grid splitter implementation
resolution = 1. / 111.
# resolution = 1.
grid = create_gridxy_global(resolution=resolution, wrapped=False, crs=ocgis.crs.Spherical())
field = create_exact_field(grid, 'exact', ntime=3, fill_data_var=False, crs=ocgis.crs.Spherical())
field.write(os.path.join(OUTDIR, 'dst_field_1km.nc'))
gs = GridChunker(grid, grid, (10, 10))
ctr = 1
for grid_sub in gs.iter_dst_grid_subsets():
subset_filename = os.path.join(OUTDIR, 'src_subset_{}.nc'.format(ctr))
dst_subset_filename = os.path.join(OUTDIR, 'dst_subset_{}.nc'.format(ctr))
if vm.rank == 0:
print 'creating subset:', subset_filename
with vm.scoped_by_emptyable('grid subset', grid_sub):
# Only return the first time slice.
SNIPPET = True
# Data returns will overwrite in this case. Use with caution!!
env.OVERWRITE = True
# This is where to find the shapfiles.
ocgis.env.DIR_GEOMCABINET = os.path.join(os.getcwd(), os.path.split(ocgis.test.__file__)[0], 'bin')
########################################################################################################################
# Write example datasets for use in this example.
grid = create_gridxy_global(resolution=3.0)
vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax']
paths = ['{}.nc'.format(ii) for ii in vars]
field_names = ['tasmin', 'tas', 'tasmax']
for idx, (path, var) in enumerate(zip(paths, vars)):
field = create_exact_field(grid.copy(), var, ntime=3)
field.data_variables[0].get_value()[:] = idx + 1
field.write(path)
# for path in paths:
# RequestDataset(path).inspect()
########################################################################################################################
# Create the request dataset objects for the file paths we'd like to subset. Variable will often be auto-discovered and
# default field names will be created. We'll just pass in everything here.
rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in
zip(paths, vars, field_names)]
########################################################################################################################
# Return in-memory as an OCGIS collection for a single geometry.
from ocgis import RequestDataset
from ocgis.test.base import create_gridxy_global, create_exact_field
from ocgis.variable.crs import Spherical
# Create synthetic data for this example.
grid = create_gridxy_global(resolution=5.0)
field = create_exact_field(grid, 'exact', ntime=31, crs=Spherical())
field.write('ocgis_example_inspect_request_dataset.nc')
# Create the request dataset object.
rd = RequestDataset('ocgis_example_inspect_request_dataset.nc')
# Provides a metadata dump for the request dataset.
rd.inspect()
# These are the auto-discovered data variable names.
assert rd.variable == 'exact'
# The dimension map provides information on how OCGIS will interpret the dataset.
rd.dimension_map.pprint()
from ocgis import SpatialCollection, GeometryVariable, Variable, crs
from ocgis.test.base import create_gridxy_global, create_exact_field
from ocgis.util.helpers import pprint_dict
from shapely.geometry import box
# Spatial collections are hierarchical and may contain groups (children). In OCGIS, the first spatial collection groups
# are subset geometries. The children or groups within the subset geometry groups contain the data subset by the
# geometry. This is easiest to understand by walking through how OCGIS subsets and assembles a spatial collection inside
# operations.
# Create a test grid.
grid = create_gridxy_global()
# Create an exact field on the grid.
field = create_exact_field(grid, 'foo')
# Create a subset geometry.
subset_geom = box(30., 20., 40., 25.)
# Subset the field using the geometry. Note we subset the grid and return its parent (the field).
sub = field.grid.get_intersects(subset_geom).parent
# Subset geometries are themselves treated as fields. Convert the geometry to an OCGIS geometry variable.
gvar = GeometryVariable.from_shapely(subset_geom, ugid=11, crs=crs.Spherical())
# Add some descriptive variables.
info = Variable(name='desc', value=['random bbox'], dtype=str, dimensions=gvar.dimensions[0])
gvar.parent.add_variable(info, is_data=True)
height = Variable(name='height', value=[30], dimensions=gvar.dimensions[0], attrs={'made_up': 'yes'})
gvar.parent.add_variable(height, is_data=True)
from ocgis.test.base import create_gridxy_global, create_exact_field # Name of the variable to subset. VAR_TAS = 'tas' # Make it easy to switch to non-snippet requests. SNIPPET = True # Set output directory for shapefile and keyed formats. (MAKE SURE IT EXISTS!) ocgis.env.DIR_OUTPUT = tempfile.mkdtemp() print ocgis.env.DIR_OUTPUT # The bounding box coordinates [minx, miny, maxx, maxy] for the state of Colorado in WGS84 latitude/longitude # coordinates. BBOX = [-109.1, 36.9, -102.0, 41.0] # Create synthetic data for this example. grid = create_gridxy_global(resolution=5.0) field = create_exact_field(grid, VAR_TAS, ntime=31) data_path = os.path.join(ocgis.env.DIR_OUTPUT, 'ocgis_example_simple_subset.nc') field.write(data_path) # This object will be reused so just build it once. Variable names are typically auto-discovered. rd = ocgis.RequestDataset(data_path, VAR_TAS) ######################################################################################################################## # Returning an OCGIS spatial collection ret = ocgis.OcgOperations(dataset=rd, geom=BBOX, snippet=SNIPPET).execute() ######################################################################################################################## # Returning conversions output_formats = ['shp', 'csv', 'csv-shp', 'nc']
from shapely.geometry import box
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
assert ocgis.vm.size_global == 2
# Create synthetic data for this example. Create and write the data on a single process. Subcommunicators are always
# given a name.
with ocgis.vm.scoped('serial write', [0]):
# There will be null communicators for the duration of the context manager.
if not ocgis.vm.is_null:
grid = create_gridxy_global(resolution=5.0, dist=False)
field = create_exact_field(grid, 'exact', ntime=31)
field.write('ocgis_parallel_example.nc')
# Place a barrier so write can finish. Race conditions can occur with subcommunicators. Generally, barriers are managed
# by the operations.
ocgis.vm.barrier()
# This is our subset geometry.
bbox = [150, 30, 170, 50]
bbox = box(*bbox)
# Load the data from file.
rd = ocgis.RequestDataset('ocgis_parallel_example.nc')
field = rd.get()
# By default, data is distributed along the largest spatial dimension. The x or longitude dimension in this case.
distributed_dimension = field.grid.dimensions[1]
bounds_local = {0: (0, 36), 1: (36, 72)}
