def test_iter_repeater(self):
var1 = Variable(name='var1', value=[1, 2, 3], dimensions='dim')
var2 = Variable(name='var2', value=[1, 2, 3], dimensions='dim')
var2.get_value()[:] *= 9
repeater = ('i_am', 'a_repeater')
itr = Iterator(var1, followers=[var2], repeaters=[repeater])
desired = [OrderedDict([('i_am', 'a_repeater'), ('var1', 1), ('var2', 9)]),
OrderedDict([('i_am', 'a_repeater'), ('var1', 2), ('var2', 18)]),
OrderedDict([('i_am', 'a_repeater'), ('var1', 3), ('var2', 27)])]
actual = list(itr)
self.assertEqual(actual, desired)
def test_get_unioned_spatial_average(self):
pa = self.get_geometryvariable()
to_weight = Variable(name='to_weight', dimensions=pa.dimensions, dtype=float)
to_weight.get_value()[:] = 5.0
pa.parent.add_variable(to_weight)
unioned = pa.get_unioned(spatial_average='to_weight')
self.assertEqual(unioned.parent[to_weight.name].get_value().tolist(), [5.0])
self.assertEqual(pa.parent[to_weight.name].get_value().shape, (2,))
self.assertEqual(unioned.dimensions, unioned.parent[to_weight.name].dimensions)
self.assertEqual(id(unioned.dimensions[0]), id(unioned.parent[to_weight.name].dimensions[0]))
def test_get_unioned_spatial_average_differing_dimensions(self):
pa = self.get_geometryvariable()
to_weight = Variable(name='to_weight', dimensions=pa.dimensions, dtype=float)
to_weight.get_value()[0] = 5.0
to_weight.get_value()[1] = 10.0
pa.parent.add_variable(to_weight)
to_weight2 = Variable(name='to_weight2',
dimensions=[Dimension('time', 10), Dimension('level', 3), pa.dimensions[0]], dtype=float)
for time_idx in range(to_weight2.shape[0]):
for level_idx in range(to_weight2.shape[1]):
to_weight2.get_value()[time_idx, level_idx] = (time_idx + 2) + (level_idx + 2) ** (level_idx + 1)
pa.parent.add_variable(to_weight2)
unioned = pa.get_unioned(spatial_average=['to_weight', 'to_weight2'])
actual = unioned.parent[to_weight2.name]
self.assertEqual(actual.shape, (10, 3, 1))
self.assertEqual(to_weight2.shape, (10, 3, 2))
self.assertNumpyAll(actual.get_value(), to_weight2.get_value()[:, :, 0].reshape(10, 3, 1))
self.assertEqual(actual.dimension_names, ('time', 'level', 'ocgis_geom_union'))
self.assertEqual(unioned.parent[to_weight.name].get_value()[0], 7.5)
def test_system_through_operations(self):
"""Test calculation through operations."""
row = Variable(name='y', value=[1, 2, 3, 4], dimensions='y')
col = Variable(name='x', value=[10, 11, 12], dimensions='x')
grid = Grid(col, row)
time = TemporalVariable(name='time', value=[1, 2], dimensions='time')
data = Variable(name='data', dimensions=[time.dimensions[0]] + list(grid.dimensions))
data.get_value()[0, :] = 1
data.get_value()[1, :] = 2
field = Field(grid=grid, time=time, is_data=data)
calc = [{'func': 'sum', 'name': 'sum'}]
ops = OcgOperations(dataset=field, calc=calc, calc_grouping='day', calc_raw=True, aggregate=True)
ret = ops.execute()
actual = ret.get_element(variable_name='sum').get_masked_value().flatten()
self.assertNumpyAll(actual, np.ma.array([12.0, 24.0]))
def test_iter_formatter(self):
def _formatter_(name, value, mask):
if value is None:
modified_value = None
else:
modified_value = value * 1000
value = str(value)
ret = [(name, value), ('modified', modified_value)]
return ret
var = Variable(name='data', value=[1, 2, 3], mask=[False, True, False], dimensions='dim')
itr = Iterator(var, formatter=_formatter_)
as_list = list(itr)
actual = as_list[1][var.name]
self.assertIsNone(actual)
self.assertEqual(as_list[2][var.name], str(var.get_value()[2]))
self.assertEqual(as_list[0]['modified'], 1000)
def create_rank_valued_netcdf(self):
rank_size = 10
size_global = vm.size_global
with vm.scoped('write rank netcdf', [0]):
if not vm.is_null:
path = self.get_temporary_file_path('dist_desired.nc')
dim = Dimension('dist_dim', rank_size * size_global)
var = Variable(name='data', dimensions=dim, attrs={'hi': 5})
for rank in range(size_global):
value = np.ones(rank_size) + (10 * (rank + 1))
bounds = (rank_size * rank, rank_size * rank + rank_size)
var.get_value()[bounds[0]: bounds[1]] = value
var.parent.attrs = {'hi_dataset_level': 'whee'}
var.write(path)
else:
path = None
path = vm.bcast(path)
return path
def test_write_variable(self):
path = self.get_temporary_file_path('foo.nc')
var = Variable(name='height', value=10.0, dimensions=[])
var.write(path)
rd = RequestDataset(path)
varin = SourcedVariable(name='height', request_dataset=rd)
self.assertEqual(varin.get_value(), var.get_value())
# Test mask persists after write.
v = Variable(name='the_mask', value=[1, 2, 3, 4], mask=[False, True, True, False], dimensions='ephemeral',
fill_value=222)
path = self.get_temporary_file_path('foo.nc')
v.write(path)
rd = RequestDataset(path, driver=DriverNetcdf)
sv = SourcedVariable(name='the_mask', request_dataset=rd)
self.assertEqual(sv.get_value().tolist(), [1, 222, 222, 4])
self.assertNumpyAll(sv.get_mask(), v.get_mask())
def test_variable_collection_scatter(self):
dest_mpi = OcgDist()
five = dest_mpi.create_dimension('five', 5, dist=True)
ten = dest_mpi.create_dimension('ten', 10)
dest_mpi.create_variable(name='five', dimensions=five)
dest_mpi.create_variable(name='all_in', dimensions=ten)
dest_mpi.create_variable(name='i_could_be_a_coordinate_system')
dest_mpi.update_dimension_bounds()
if MPI_RANK == 0:
var = Variable('holds_five', np.arange(5), dimensions='five')
var_empty = Variable('i_could_be_a_coordinate_system', attrs={'reality': 'im_not'})
var_not_dist = Variable('all_in', value=np.arange(10) + 10, dimensions='ten')
vc = VariableCollection(variables=[var, var_empty, var_not_dist])
else:
vc = None
svc = variable_collection_scatter(vc, dest_mpi)
self.assertEqual(svc['i_could_be_a_coordinate_system'].attrs['reality'], 'im_not')
if MPI_RANK < 2:
self.assertFalse(svc['all_in'].is_empty)
self.assertNumpyAll(svc['all_in'].get_value(), np.arange(10) + 10)
self.assertFalse(svc.is_empty)
self.assertFalse(svc['i_could_be_a_coordinate_system'].is_empty)
else:
self.assertTrue(svc['all_in'].is_empty)
self.assertTrue(svc.is_empty)
self.assertTrue(svc['i_could_be_a_coordinate_system'].is_empty)
if MPI_RANK == 0:
self.assertNumpyAll(var.get_value(), vc[var.name].get_value())
actual = svc['holds_five'].get_value()
if MPI_SIZE == 2:
desired = {0: np.arange(3), 1: np.arange(3, 5)}
self.assertNumpyAll(actual, desired[MPI_RANK])
actual = svc['holds_five'].is_empty
if MPI_RANK > 1:
self.assertTrue(actual)
else:
self.assertFalse(actual)
def _convert_to_ugrid_(field):
"""
Takes field data out of the OCGIS unstructured format (similar to UGRID) converting to the format expected
by ESMF Unstructured metadata.
"""
# The driver for the current field must be NetCDF UGRID to ensure interpretability.
assert field.dimension_map.get_driver() == DriverKey.NETCDF_UGRID
grid = field.grid
# Three-dimensional data is not supported.
assert not grid.has_z
# Number of coordinate dimension. This will be 3 for three-dimensional data.
coord_dim = Dimension('coordDim', 2)
# Transform ragged array to one-dimensional array. #############################################################
cindex = grid.cindex
elements = cindex.get_value()
num_element_conn_data = [e.shape[0] for e in elements.flat]
length_connection_count = sum(num_element_conn_data)
esmf_element_conn = np.zeros(length_connection_count, dtype=elements[0].dtype)
start = 0
tag_start_index = MPITag.START_INDEX
# Collapse the ragged element index array into a single dimensioned vector. This communication block finds the
# size for the new array. ######################################################################################
if vm.size > 1:
max_index = max([ii.max() for ii in elements.flat])
if vm.rank == 0:
vm.comm.isend(max_index + 1, dest=1, tag=tag_start_index)
adjust = 0
else:
adjust = vm.comm.irecv(source=vm.rank - 1, tag=tag_start_index)
adjust = adjust.wait()
if vm.rank != vm.size - 1:
vm.comm.isend(max_index + 1 + adjust, dest=vm.rank + 1, tag=tag_start_index)
# Fill the new vector for the element connectivity. ############################################################
for ii in elements.flat:
if vm.size > 1:
if grid.archetype.has_multi:
mbv = cindex.attrs[OcgisConvention.Name.MULTI_BREAK_VALUE]
replace_breaks = np.where(ii == mbv)[0]
else:
replace_breaks = []
ii = ii + adjust
if len(replace_breaks) > 0:
ii[replace_breaks] = mbv
esmf_element_conn[start: start + ii.shape[0]] = ii
start += ii.shape[0]
# Create the new data representation. ##########################################################################
connection_count = create_distributed_dimension(esmf_element_conn.size, name='connectionCount')
esmf_element_conn_var = Variable(name='elementConn', value=esmf_element_conn, dimensions=connection_count,
dtype=np.int32)
esmf_element_conn_var.attrs[CFName.LONG_NAME] = 'Node indices that define the element connectivity.'
mbv = cindex.attrs.get(OcgisConvention.Name.MULTI_BREAK_VALUE)
if mbv is not None:
esmf_element_conn_var.attrs['polygon_break_value'] = mbv
esmf_element_conn_var.attrs['start_index'] = grid.start_index
ret = VariableCollection(variables=field.copy().values(), force=True)
# Rename the element count dimension.
original_name = ret[cindex.name].dimensions[0].name
ret.rename_dimension(original_name, 'elementCount')
# Add the element-node connectivity variable to the collection.
ret.add_variable(esmf_element_conn_var)
num_element_conn = Variable(name='numElementConn',
value=num_element_conn_data,
dimensions=cindex.dimensions[0],
attrs={CFName.LONG_NAME: 'Number of nodes per element.'},
dtype=np.int32)
ret.add_variable(num_element_conn)
# Check that the node count dimension is appropriately named.
gn_name = grid.node_dim.name
if gn_name != 'nodeCount':
ret.dimensions[gn_name] = ret.dimensions[gn_name].copy()
ret.rename_dimension(gn_name, 'nodeCount')
node_coords = Variable(name='nodeCoords', dimensions=(ret.dimensions['nodeCount'], coord_dim))
node_coords.units = 'degrees'
node_coords.attrs[CFName.LONG_NAME] = 'Node coordinate values indexed by element connectivity.'
node_coords.attrs['coordinates'] = 'x y'
fill = node_coords.get_value()
fill[:, 0] = grid.x.get_value()
fill[:, 1] = grid.y.get_value()
ret.pop(grid.x.name)
ret.pop(grid.y.name)
ret.add_variable(node_coords)
ret.attrs['gridType'] = 'unstructured'
ret.attrs['version'] = '0.9'
# Remove the coordinate index, this does not matter.
if field.grid.cindex is not None:
ret.remove_variable(field.grid.cindex.name)
return ret