def test_compute_with_geom(self):
grid = create_gridxy_global(resolution=5.0)
field = create_exact_field(grid, 'exact')
path = self.get_temporary_file_path('foo.nc')
field.write(path)
rd = RequestDataset(path)
calcs = [[{'func': 'mean', 'name': 'mean'}], None]
for ii, c in enumerate(calcs):
ops = ocgis.OcgOperations(dataset=rd, calc=c, calc_grouping=['month'], output_format='nc',
geom=self.path_state_boundaries, select_ugid=[2, 9, 12, 23, 25],
add_auxiliary_files=False, agg_selection=True, prefix=str(ii) + '_foo')
ret = compute(ops, 5, verbose=False)
ops.prefix = str(ii)
ret_ocgis = ops.execute()
self.assertNcEqual(ret, ret_ocgis, check_fill_value=False, check_types=False,
ignore_attributes={'global': ['history'], 'mean': ['_FillValue']})
def test_write_esmf_weights(self):
# Create source and destination fields. This is the identity test, so the source and destination fields are
# equivalent.
src_grid = create_gridxy_global(resolution=3.0, crs=Spherical())
# Only test masking in serial to make indexing easier...just being lazy
if vm.size == 1:
mask = src_grid.get_mask(create=True)
mask[4, 5] = True
mask[25, 27] = True
src_grid.set_mask(mask)
self.assertEqual(src_grid.get_mask().sum(), 2)
src_field = create_exact_field(src_grid, 'foo', ntime=3)
dst_field = deepcopy(src_field)
# Write the fields to disk for use in global file reconstruction and testing.
if vm.rank == 0:
master_path = self.get_temporary_file_path('foo.nc')
src_field_path = self.get_temporary_file_path('src_field.nc')
else:
master_path = None
src_field_path = None
master_path = vm.bcast(master_path)
src_field_path = vm.bcast(src_field_path)
assert not os.path.exists(master_path)
dst_field.write(master_path)
src_field.write(src_field_path)
# Remove the destination data variable to test its creation and filling
dst_field.remove_variable('foo')
# Chunk the fields and generate weights
paths = {'wd': self.current_dir_output}
gc = GridChunker(src_field, dst_field, nchunks_dst=(2, 2), genweights=True, paths=paths,
esmf_kwargs={'regrid_method': 'BILINEAR'})
gc.write_chunks()
# This is the path to the index file describing how to reconstruct the grid file
index_path = os.path.join(self.current_dir_output, gc.paths['index_file'])
# Execute the sparse matrix multiplication using weights read from file
gc.smm(index_path, paths['wd'])
with vm.scoped('index and reconstruct', [0]):
if not vm.is_null:
# Reconstruct the global destination file
gc.insert_weighted(index_path, self.current_dir_output, master_path)
# Load the actual values from file (destination)
actual_field = RequestDataset(master_path).create_field()
actual = actual_field.data_variables[0].mv()
# Load the desired data from file (original values in the source field)
desired = RequestDataset(src_field_path).create_field().data_variables[0].mv()
if vm.size_global == 1: # Masking is only tested in serial
self.assertEqual(actual_field.grid.get_mask().sum(), 2)
else:
self.assertIsNone(actual_field.grid.get_mask())
self.assertNumpyAll(actual, desired)
import tempfile
import dask
import ocgis
from ocgis.spatial.grid_chunker import GridChunker
from ocgis.test import create_gridxy_global, create_exact_field
import numpy as np
OUTDIR = tempfile.gettempdir()
GRIDFN = os.path.join(OUTDIR, 'grid.nc')
# Create a test grid
grid = create_gridxy_global(crs=ocgis.crs.Spherical())
field = create_exact_field(grid, 'foo')
field.write(GRIDFN)
def apply_by_spatial_chunk(src_filename, dst_filename, nchunks, chunk_idx, **kwargs):
"""
Create a spatial chunk from source and destination CF-Grid NetCDF files. Each source and destination chunk is
converted to a :class:`xarray.Dataset`. See :class:`~ocgis.spatial.grid_chunker.GridChunker` for more documentation
on the spatial chunking.
Returns `0` if the chunking is successful.
:param str src_filename: Path to source NetCDF file.
:param str dst_filename: Path to destination NetCDF file.
:param nchunks: The chunking decomposition for the destination grid. See :class:`~ocgis.spatial.grid_chunker.GridChunker`.
:type nchunks: tuple(int, ...)
from ocgis import OcgOperations
from ocgis.test import create_gridxy_global, create_exact_field
from ocgis.variable import stack
# Create data files using the CF-Grid metadata convention with three timesteps per file ################################
filenames = []
for ii in range(1, 4):
grid = create_gridxy_global()
field = create_exact_field(grid, 'data', ntime=3)
field.time.v()[:] += 10 * ii
field['data'].v()[:] += 10 * ii
currfn = 'ocgis_example_stacking_subsets_{}.nc'.format(ii)
filenames.append(currfn)
field.write(currfn)
########################################################################################################################
# Subset each file created above using a bounding box and return the data as a spatial collection.
colls = [OcgOperations(dataset={'uri': fn}, geom=[40, 30, 50, 60]).execute() for fn in filenames]
# Extract the fields to stack from each spatial collection.
colls_to_stack = [coll.get_element() for coll in colls]
# Stack the data along the time dimension returning a field.
stacked = stack(colls_to_stack, 'time')
def test_write_esmf_weights(self):
# Create source and destination fields. This is the identity test, so the source and destination fields are
# equivalent.
src_grid = create_gridxy_global(resolution=3.0, crs=Spherical())
# Only test masking in serial to make indexing easier...just being lazy
if vm.size == 1:
mask = src_grid.get_mask(create=True)
mask[4, 5] = True
mask[25, 27] = True
src_grid.set_mask(mask)
self.assertEqual(src_grid.get_mask().sum(), 2)
src_field = create_exact_field(src_grid, 'foo', ntime=3)
dst_field = deepcopy(src_field)
# Write the fields to disk for use in global file reconstruction and testing.
if vm.rank == 0:
master_path = self.get_temporary_file_path('foo.nc')
src_field_path = self.get_temporary_file_path('src_field.nc')
else:
master_path = None
src_field_path = None
master_path = vm.bcast(master_path)
src_field_path = vm.bcast(src_field_path)
assert not os.path.exists(master_path)
dst_field.write(master_path)
src_field.write(src_field_path)
# Remove the destination data variable to test its creation and filling
dst_field.remove_variable('foo')
# Chunk the fields and generate weights
paths = {'wd': self.current_dir_output}
gc = GridChunker(src_field,
dst_field,
nchunks_dst=(2, 2),
genweights=True,
paths=paths,
esmf_kwargs={'regrid_method': 'BILINEAR'})
gc.write_chunks()
# This is the path to the index file describing how to reconstruct the grid file
index_path = os.path.join(self.current_dir_output,
gc.paths['index_file'])
# Execute the sparse matrix multiplication using weights read from file
gc.smm(index_path, paths['wd'])
with vm.scoped('index and reconstruct', [0]):
if not vm.is_null:
# Reconstruct the global destination file
gc.insert_weighted(index_path, self.current_dir_output,
master_path)
# Load the actual values from file (destination)
actual_field = RequestDataset(master_path).create_field()
actual = actual_field.data_variables[0].mv()
# Load the desired data from file (original values in the source field)
desired = RequestDataset(
src_field_path).create_field().data_variables[0].mv()
if vm.size_global == 1: # Masking is only tested in serial
self.assertEqual(actual_field.grid.get_mask().sum(), 2)
else:
self.assertIsNone(actual_field.grid.get_mask())
self.assertNumpyAll(actual, desired)
from ocgis.test import create_gridxy_global, create_exact_field # Create an in-memory field. grid = create_gridxy_global() field = create_exact_field(grid, 'a_var') # Set the test variable's units. field['a_var'].units = 'odd_source_units' # Another way to set the source units... # field['a_var'].attrs['units'] = 'odd_source_units' # These calls retrieve the underlying data values without a mask. # value = field['a_var'].v() # value = field['a_var'].get_value() # It is best to work with the masked values unless performance is an issue. masked_value = field['a_var'].mv() # masked_value = field['a_var'].get_masked_value() # Convert the units in-place and update the units attribute on the target variable. masked_value[:] *= 1.75 field['a_var'].units = 'odd_destination_units'