def chunked_smm(wd, index_path, insert_weighted, destination, data_variables):
if wd is None:
wd = os.getcwd()
if data_variables != 'auto':
data_variables = data_variables.split(',')
if index_path is None:
index_path = os.path.join(
wd, constants.GridChunkerConstants.DEFAULT_PATHS['index_file'])
ocgis.vm.barrier()
assert os.path.exists(index_path)
if insert_weighted:
if destination is None:
raise ValueError(
'If --insert_weighted, then "destination" must be provided.')
# ------------------------------------------------------------------------------------------------------------------
GridChunker.smm(index_path, wd, data_variables=data_variables)
if insert_weighted:
with ocgis.vm.scoped_barrier(first=True, last=True):
with ocgis.vm.scoped('insert weighted', [0]):
if not ocgis.vm.is_null:
GridChunker.insert_weighted(index_path,
wd,
destination,
data_variables=data_variables)
def chunked_smm(wd, index_path, insert_weighted, destination, data_variables):
if wd is None:
wd = os.getcwd()
if data_variables != 'auto':
data_variables = data_variables.split(',')
if index_path is None:
index_path = os.path.join(wd, constants.GridChunkerConstants.DEFAULT_PATHS['index_file'])
ocgis.vm.barrier()
assert os.path.exists(index_path)
if insert_weighted:
if destination is None:
raise ValueError('If --insert_weighted, then "destination" must be provided.')
# ------------------------------------------------------------------------------------------------------------------
GridChunker.smm(index_path, wd, data_variables=data_variables)
if insert_weighted:
with ocgis.vm.scoped_barrier(first=True, last=True):
with ocgis.vm.scoped('insert weighted', [0]):
if not ocgis.vm.is_null:
GridChunker.insert_weighted(index_path, wd, destination, data_variables=data_variables)
def test_write_esmf_weights(self):
# Create source and destination fields. This is the identity test, so the source and destination fields are
# equivalent.
src_grid = create_gridxy_global(resolution=3.0, crs=Spherical())
# Only test masking in serial to make indexing easier...just being lazy
if vm.size == 1:
mask = src_grid.get_mask(create=True)
mask[4, 5] = True
mask[25, 27] = True
src_grid.set_mask(mask)
self.assertEqual(src_grid.get_mask().sum(), 2)
src_field = create_exact_field(src_grid, 'foo', ntime=3)
dst_field = deepcopy(src_field)
# Write the fields to disk for use in global file reconstruction and testing.
if vm.rank == 0:
master_path = self.get_temporary_file_path('foo.nc')
src_field_path = self.get_temporary_file_path('src_field.nc')
else:
master_path = None
src_field_path = None
master_path = vm.bcast(master_path)
src_field_path = vm.bcast(src_field_path)
assert not os.path.exists(master_path)
dst_field.write(master_path)
src_field.write(src_field_path)
# Remove the destination data variable to test its creation and filling
dst_field.remove_variable('foo')
# Chunk the fields and generate weights
paths = {'wd': self.current_dir_output}
gc = GridChunker(src_field, dst_field, nchunks_dst=(2, 2), genweights=True, paths=paths,
esmf_kwargs={'regrid_method': 'BILINEAR'})
gc.write_chunks()
# This is the path to the index file describing how to reconstruct the grid file
index_path = os.path.join(self.current_dir_output, gc.paths['index_file'])
# Execute the sparse matrix multiplication using weights read from file
gc.smm(index_path, paths['wd'])
with vm.scoped('index and reconstruct', [0]):
if not vm.is_null:
# Reconstruct the global destination file
gc.insert_weighted(index_path, self.current_dir_output, master_path)
# Load the actual values from file (destination)
actual_field = RequestDataset(master_path).create_field()
actual = actual_field.data_variables[0].mv()
# Load the desired data from file (original values in the source field)
desired = RequestDataset(src_field_path).create_field().data_variables[0].mv()
if vm.size_global == 1: # Masking is only tested in serial
self.assertEqual(actual_field.grid.get_mask().sum(), 2)
else:
self.assertIsNone(actual_field.grid.get_mask())
self.assertNumpyAll(actual, desired)
def test_write_esmf_weights(self):
# Create source and destination fields. This is the identity test, so the source and destination fields are
# equivalent.
src_grid = create_gridxy_global(resolution=3.0, crs=Spherical())
# Only test masking in serial to make indexing easier...just being lazy
if vm.size == 1:
mask = src_grid.get_mask(create=True)
mask[4, 5] = True
mask[25, 27] = True
src_grid.set_mask(mask)
self.assertEqual(src_grid.get_mask().sum(), 2)
src_field = create_exact_field(src_grid, 'foo', ntime=3)
dst_field = deepcopy(src_field)
# Write the fields to disk for use in global file reconstruction and testing.
if vm.rank == 0:
master_path = self.get_temporary_file_path('foo.nc')
src_field_path = self.get_temporary_file_path('src_field.nc')
else:
master_path = None
src_field_path = None
master_path = vm.bcast(master_path)
src_field_path = vm.bcast(src_field_path)
assert not os.path.exists(master_path)
dst_field.write(master_path)
src_field.write(src_field_path)
# Remove the destination data variable to test its creation and filling
dst_field.remove_variable('foo')
# Chunk the fields and generate weights
paths = {'wd': self.current_dir_output}
gc = GridChunker(src_field,
dst_field,
nchunks_dst=(2, 2),
genweights=True,
paths=paths,
esmf_kwargs={'regrid_method': 'BILINEAR'})
gc.write_chunks()
# This is the path to the index file describing how to reconstruct the grid file
index_path = os.path.join(self.current_dir_output,
gc.paths['index_file'])
# Execute the sparse matrix multiplication using weights read from file
gc.smm(index_path, paths['wd'])
with vm.scoped('index and reconstruct', [0]):
if not vm.is_null:
# Reconstruct the global destination file
gc.insert_weighted(index_path, self.current_dir_output,
master_path)
# Load the actual values from file (destination)
actual_field = RequestDataset(master_path).create_field()
actual = actual_field.data_variables[0].mv()
# Load the desired data from file (original values in the source field)
desired = RequestDataset(
src_field_path).create_field().data_variables[0].mv()
if vm.size_global == 1: # Masking is only tested in serial
self.assertEqual(actual_field.grid.get_mask().sum(), 2)
else:
self.assertIsNone(actual_field.grid.get_mask())
self.assertNumpyAll(actual, desired)