def test_write_variable_collection_object_arrays(self):
"""Test writing variable length arrays in parallel."""
with vm.scoped('write', [0]):
if not vm.is_null:
path_actual = self.get_temporary_file_path('in.nc')
path_desired = self.get_temporary_file_path('out.nc')
value = [[1, 3, 5],
[7, 9],
[11]]
v = Variable(name='objects', value=value, fill_value=4, dtype=ObjectType(int), dimensions='values')
v.write(path_desired)
else:
v, path_actual, path_desired = [None] * 3
path_actual = MPI_COMM.bcast(path_actual)
path_desired = MPI_COMM.bcast(path_desired)
dest_mpi = OcgDist()
dest_mpi.create_dimension('values', 3, dist=True)
dest_mpi.update_dimension_bounds()
scattered = variable_scatter(v, dest_mpi)
outvc = VariableCollection(variables=[scattered])
with vm.scoped_by_emptyable('write', outvc):
if not vm.is_null:
outvc.write(path_actual)
if MPI_RANK == 0:
self.assertNcEqual(path_actual, path_desired)
def test_write_variable_collection_object_arrays(self):
"""Test writing variable length arrays in parallel."""
with vm.scoped('write', [0]):
if not vm.is_null:
path_actual = self.get_temporary_file_path('in.nc')
path_desired = self.get_temporary_file_path('out.nc')
value = [[1, 3, 5], [7, 9], [11]]
v = Variable(name='objects',
value=value,
fill_value=4,
dtype=ObjectType(int),
dimensions='values')
v.write(path_desired)
else:
v, path_actual, path_desired = [None] * 3
path_actual = MPI_COMM.bcast(path_actual)
path_desired = MPI_COMM.bcast(path_desired)
dest_mpi = OcgDist()
dest_mpi.create_dimension('values', 3, dist=True)
dest_mpi.update_dimension_bounds()
scattered = variable_scatter(v, dest_mpi)
outvc = VariableCollection(variables=[scattered])
with vm.scoped_by_emptyable('write', outvc):
if not vm.is_null:
outvc.write(path_actual)
if MPI_RANK == 0:
self.assertNcEqual(path_actual, path_desired)
def test_remove_netcdf_attribute(self):
path = self.get_temporary_file_path('foo.nc')
var = Variable(name='test', attrs={'remove_me': 10})
var.write(path)
remove_netcdf_attribute(path, var.name, 'remove_me')
with self.nc_scope(path) as ds:
actual = ds.variables[var.name]
self.assertFalse(hasattr(actual, 'remove_me'))
def test_remove_netcdf_attribute(self):
path = self.get_temporary_file_path('foo.nc')
var = Variable(name='test', attrs={'remove_me': 10})
var.write(path)
remove_netcdf_attribute(path, var.name, 'remove_me')
with self.nc_scope(path) as ds:
actual = ds.variables[var.name]
self.assertFalse(hasattr(actual, 'remove_me'))
def test_system_netcdf_output_format(self):
path = self.get_temporary_file_path('foo.nc')
var = Variable('vec', value=[1, 2, 3, 4, 5], dimensions='dvec', dtype=np.int32)
var.write(path)
with self.nc_scope(path, 'r') as ds:
self.assertEqual(ds.data_model, 'NETCDF4')
rd = RequestDataset(uri=path)
ops = OcgOperations(dataset=rd, prefix='converted', output_format='nc',
output_format_options={'data_model': 'NETCDF4_CLASSIC'})
ret = ops.execute()
with self.nc_scope(ret, 'r') as ds:
self.assertEqual(ds.data_model, 'NETCDF4_CLASSIC')
def create_rank_valued_netcdf(self):
rank_size = 10
size_global = vm.size_global
with vm.scoped('write rank netcdf', [0]):
if not vm.is_null:
path = self.get_temporary_file_path('dist_desired.nc')
dim = Dimension('dist_dim', rank_size * size_global)
var = Variable(name='data', dimensions=dim, attrs={'hi': 5})
for rank in range(size_global):
value = np.ones(rank_size) + (10 * (rank + 1))
bounds = (rank_size * rank, rank_size * rank + rank_size)
var.get_value()[bounds[0]: bounds[1]] = value
var.parent.attrs = {'hi_dataset_level': 'whee'}
var.write(path)
else:
path = None
path = vm.bcast(path)
return path
def test_write_variable(self):
path = self.get_temporary_file_path('foo.nc')
var = Variable(name='height', value=10.0, dimensions=[])
var.write(path)
rd = RequestDataset(path)
varin = SourcedVariable(name='height', request_dataset=rd)
self.assertEqual(varin.get_value(), var.get_value())
# Test mask persists after write.
v = Variable(name='the_mask', value=[1, 2, 3, 4], mask=[False, True, True, False], dimensions='ephemeral',
fill_value=222)
path = self.get_temporary_file_path('foo.nc')
v.write(path)
rd = RequestDataset(path, driver=DriverNetcdf)
sv = SourcedVariable(name='the_mask', request_dataset=rd)
self.assertEqual(sv.get_value().tolist(), [1, 222, 222, 4])
self.assertNumpyAll(sv.get_mask(), v.get_mask())
def test_write_variable(self):
path = self.get_temporary_file_path('foo.nc')
var = Variable(name='height', value=10.0, dimensions=[])
var.write(path)
rd = RequestDataset(path)
varin = SourcedVariable(name='height', request_dataset=rd)
self.assertEqual(varin.get_value(), var.get_value())
# Test mask persists after write.
v = Variable(name='the_mask',
value=[1, 2, 3, 4],
mask=[False, True, True, False],
dimensions='ephemeral',
fill_value=222)
path = self.get_temporary_file_path('foo.nc')
v.write(path)
rd = RequestDataset(path, driver=DriverNetcdf)
sv = SourcedVariable(name='the_mask', request_dataset=rd)
self.assertEqual(sv.get_value().tolist(), [1, 222, 222, 4])
self.assertNumpyAll(sv.get_mask(), v.get_mask())