def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_system_with_time_data(self):
"""Test writing data with a time dimension."""
path = self.get_temporary_file_path('what.shp')
t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time')
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)],
name='geom',
dimensions='time')
field = Field(variables=[t, geom],
dimension_map={
'time': {
'variable': 'time'
},
'geom': {
'variable': 'geom'
}
})
field.write(path,
iter_kwargs={'variable': 'time'},
driver=DriverVector)
rd = RequestDataset(uri=path)
field2 = rd.get()
# netcdftime worthlessness
poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'],
['1-01-02 12:00:00', '1-01-03 12:00:00']]
actual = field2['TIME'].get_value().tolist()
res = [p == actual for p in poss]
self.assertTrue(any(res))
def get_ocgfield_example(self):
dtime = Dimension(name='time')
t = TemporalVariable(value=[1, 2, 3, 4], name='the_time', dimensions=dtime, dtype=float)
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
lon = Variable(value=[30., 40., 50., 60.], name='longitude', dimensions='lon')
lat = Variable(value=[-10., -20., -30., -40., -50.], name='latitude', dimensions='lat')
tas_shape = [t.shape[0], lat.shape[0], lon.shape[0]]
tas = Variable(value=np.arange(reduce_multiply(tas_shape)).reshape(*tas_shape),
dimensions=(dtime, 'lat', 'lon'), name='tas')
time_related = Variable(value=[7, 8, 9, 10], name='time_related', dimensions=dtime)
garbage1 = Variable(value=[66, 67, 68], dimensions='three', name='garbage1')
dmap = {'time': {'variable': t.name},
'x': {'variable': lon.name, DimensionMapKey.DIMENSION: [lon.dimensions[0].name]},
'y': {'variable': lat.name, DimensionMapKey.DIMENSION: [lat.dimensions[0].name]}}
field = Field(variables=[t, lon, lat, tas, garbage1, time_related], dimension_map=dmap, is_data=tas.name)
return field
def get_field(self, ntime=2, variable_name='foo', nrow=2, ncol=2):
"""Create random field where mean varies with radius and std with the
angle around the center of the grid.
"""
np.random.seed(1)
row = Variable(value=np.arange(nrow) - nrow / 2., name='row', dimensions='row')
col = Variable(value=np.arange(ncol) - ncol / 2., name='col', dimensions='col')
grid = Grid(col, row)
x, y = grid.get_value_stacked()
start = dt.datetime(2000, 1, 1)
delta = dt.timedelta(days=1)
value_temporal = [start + i * delta for i in range(ntime)]
temporal = TemporalVariable(value=value_temporal, dimensions='time',
name='time')
nlevel = 1
level = None
nrlz = 1
realization = None
value = np.random.rand(nrlz, ntime, nlevel, nrow, ncol) * np.arctan2(x, y).clip(.1) + np.hypot(x, y)
variable = Variable(name=variable_name,
value=value,
dimensions=['realization', 'time', 'level', 'row',
'col'])
field = Field(grid=grid, time=temporal, is_data=variable, level=level,
realization=realization)
return field
def test_system_get_field_dimensioned_variables(self):
"""Test data is appropriately tagged to identify dimensioned variables."""
path = self.get_temporary_file_path('foo.nc')
time = TemporalVariable(value=[1, 2, 3], dimensions='time')
x = Variable(name='x', value=[10, 20], dimensions='x')
y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y')
data1 = Variable(name='data1',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data2 = Variable(name='data2',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time'])
field = Field(time=time,
grid=Grid(x, y),
variables=[data1, data2, data3])
field.write(path)
# Test dimensioned variables are read from a file with appropriate metadata.
rd = RequestDataset(path)
self.assertEqual(rd.variable, ('data1', 'data2'))
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data1', 'data2'))
# Test dimensioned variables are overloaded.
rd = RequestDataset(path, variable='data2')
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data2', ))
def get_wrap_field(crs=None, unwrapped=True):
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=False)
ompi.create_dimension('y', 7, dist=True)
ompi.create_dimension('time', size_current=4, dist=False)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
row = Variable(value=[-60, -40, -20, 0, 20, 40, 60], name='y', dimensions='y')
if unwrapped:
col_value = [1, 90, 180, 225, 270]
else:
col_value = [-170, -85, 0, 85, 170]
col = Variable(value=col_value, name='x', dimensions='x')
grid = Grid(col, row)
value = np.zeros((4, 7, 5))
for col_idx in range(value.shape[-1]):
value[:, :, col_idx] = col_idx
time = TemporalVariable(name='time', value=[1, 2, 3, 4], dimensions='time')
var = Variable(name='foo', value=value, dimensions=['time', 'y', 'x'])
field = Field(grid=grid, is_data=var, crs=crs, time=time)
else:
field = None
field = variable_collection_scatter(field, ompi)
return field
def add_data_variable_to_grid(grid):
ydim, xdim = grid.dimensions
tdim = Dimension(name='time', size=None)
value = np.random.rand(31, ydim.size, xdim.size)
data = Variable(name='data', dimensions=[tdim, ydim, xdim], value=value)
tvar = TemporalVariable(name='time', value=list(range(31)), dimensions=tdim, attrs={'axis': 'T'})
grid.parent.add_variable(data)
grid.parent.add_variable(tvar)
def test_system_dimension_map_formatting(self):
"""Test any formatting of the incoming dimension map by the field."""
dmap = {'time': {'variable': 'time'}}
time = TemporalVariable(name='time', value=[1, 2, 3], dimensions='the_time')
field = Field(time=time, dimension_map=dmap)
actual = field.dimension_map.get_dimension('time')
self.assertEqual(actual, ['the_time'])
def test_system_parallel_write_ndvariable(self):
"""Test a parallel CSV write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.csv')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
vc = VariableCollection(variables=[t, extra, x, y, data])
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, iter_kwargs={'variable': 'data', 'followers': ['time', 'extra', 'y', 'x']},
driver=DriverCSV)
if MPI_RANK == 0:
desired = 169
with open(path, 'r') as f:
lines = f.readlines()
self.assertEqual(len(lines), desired)
def test_iter_masking_and_driver(self):
"""Test mask is set to None."""
time = TemporalVariable(value=[3, 4, 5], dimensions='time')
data = Variable(value=[7, 8, 9], name='data', mask=[False, True, False], dimensions='time')
field = Field(time=time, is_data=data, variables=data)
itr = field.iter(allow_masked=True)
actual = list(itr)
self.assertIsNone(actual[1][1][data.name])
def time(self):
"""
:return: Get the field's time variable. Return ``None`` if no time is assigned.
:rtype: :class:`~ocgis.TemporalVariable` | ``None``
"""
ret = get_field_property(self, 'time')
if ret is not None:
if not isinstance(ret, TemporalGroupVariable):
ret = TemporalVariable.from_variable(ret, format_time=self.format_time)
return ret
def test_system_properties(self):
"""Test field properties."""
time = TemporalVariable(value=[20, 30, 40], dimensions=['the_time'], dtype=float, name='time')
time_bounds = TemporalVariable(value=[[15, 25], [25, 35], [35, 45]], dimensions=['times', 'bounds'],
dtype=float, name='time_bounds')
other = Variable(value=[44, 55, 66], name='other', dimensions=['times_again'])
x = Variable(value=[1, 2, 3], name='xc', dimensions=['x'])
y = Variable(value=[10, 20, 30, 40], name='yc', dimensions=['y'])
crs = CoordinateReferenceSystem(epsg=2136)
f = self.get_ocgfield(variables=[time, time_bounds, other, x, y])
f2 = deepcopy(f)
self.assertIsNone(f.realization)
self.assertIsNone(f.time)
f.dimension_map.set_variable('time', time.name)
self.assertNumpyAll(f.time.get_value(), time.get_value())
self.assertEqual(f.time.attrs['axis'], 'T')
self.assertIsNone(f.time.bounds)
f.dimension_map.set_variable('time', 'time', bounds=time_bounds.name)
self.assertNumpyAll(f.time.bounds.get_value(), time_bounds.get_value())
self.assertIn('other', f.time.parent)
dims = f.dimension_map.get_dimension('time')
dims += ['times', 'times_again', 'the_time']
sub = f.get_field_slice({'time': slice(1, 2)})
desired = OrderedDict([('time', (1,)), ('time_bounds', (1, 2)), ('other', (1,)), ('xc', (3,)), ('yc', (4,))])
self.assertEqual(sub.shapes, desired)
self.assertIsNone(sub.grid)
sub.dimension_map.set_variable('x', 'xc')
sub.dimension_map.set_variable('y', 'yc')
# Test writing to netCDF will load attributes.
path = self.get_temporary_file_path('foo.nc')
sub.write(path)
with self.nc_scope(path) as ds:
self.assertEqual(ds.variables[x.name].axis, 'X')
self.assertEqual(ds.variables[y.name].axis, 'Y')
self.assertEqual(sub.x.attrs['axis'], 'X')
self.assertEqual(sub.y.attrs['axis'], 'Y')
self.assertIsInstance(sub.grid, Grid)
desired = OrderedDict([('time', (1,)), ('time_bounds', (1, 2)), ('other', (1,)), ('xc', (3,)), ('yc', (4,))])
self.assertEqual(sub.shapes, desired)
# Test a subset.
bbox = [1.5, 15, 2.5, 35]
data = Variable(name='data', value=np.random.rand(3, 4), dimensions=['x', 'y'])
f2.add_variable(data)
f2.dimension_map.set_variable('x', 'xc')
f2.dimension_map.set_variable('y', 'yc')
bbox = box(*bbox)
spatial_sub = f2.grid.get_intersects(bbox).parent
desired = OrderedDict([('time', (3,)), ('time_bounds', (3, 2)), ('other', (3,)), ('xc', (1,)), ('yc', (2,)),
('data', (1, 2)), ])
self.assertEqual(spatial_sub.shapes, desired)
def init_variable_value(self, variable):
"""Set the variable value from source data conforming units in the process."""
from ocgis.variable.temporal import TemporalVariable
value = self.get_variable_value(variable)
variable.set_value(value, update_mask=True)
# Conform the units if requested. Need to check if this variable is inside a group to find the appropriate
# metadata.
meta = get_variable_metadata_from_request_dataset(self, variable)
conform_units_to = meta.get('conform_units_to')
if conform_units_to is not None:
# The initialized units for the variable are overloaded by the destination / conform to units.
if isinstance(variable, TemporalVariable):
from_units = TemporalVariable(units=meta['attrs']['units'],
calendar=meta['attrs'].get('calendar'))
from_units = from_units.cfunits
else:
from_units = meta['attrs']['units']
variable.cfunits_conform(conform_units_to, from_units=from_units)
def test_system_request_dataset_modifiers(self):
"""
Test request dataset arguments are applied in operations to fields. There are parameters that may be passed to a
request dataset or used solely in operations.
"""
def _the_func_(arr, bounds=None):
tfret = []
for ctr, element in enumerate(arr.flat):
if element.year == 2004:
tfret.append(ctr)
return tfret
time = TemporalVariable(name='my_time', value=[400, 800, 1200, 1600, 2000, 2400], dimensions='time_dimension',
units='days since 2001-1-1')
level = Variable(name='my_level', value=[20, 30, 40, 50], dimensions='level_dimension')
np.random.seed(1)
original_value = np.random.rand(time.shape[0], level.shape[0])
data = Variable(name='data',
value=original_value,
dimensions=['time_dimension', 'level_dimension'],
units='fahrenheit')
field = Field(time=time, level=level, is_data=data)
ops = OcgOperations(dataset=field,
time_range=[datetime.datetime(2003, 1, 1), datetime.datetime(2007, 1, 1)],
time_region={'year': [2003, 2004, 2005]},
time_subset_func=_the_func_,
level_range=[30, 40],
conform_units_to='celsius')
ret = ops.execute()
actual = ret.get_element()
self.assertEqual(actual.time.shape, (1,))
self.assertEqual(actual.time.value_datetime[0].year, 2004)
self.assertEqual(actual.level.get_value().tolist(), [30, 40])
self.assertAlmostEqual(actual.data_variables[0].get_value().mean(), -17.511663542109229)
self.assertEqual(actual.data_variables[0].units, 'celsius')
def test_system_through_operations(self):
"""Test calculation through operations."""
row = Variable(name='y', value=[1, 2, 3, 4], dimensions='y')
col = Variable(name='x', value=[10, 11, 12], dimensions='x')
grid = Grid(col, row)
time = TemporalVariable(name='time', value=[1, 2], dimensions='time')
data = Variable(name='data',
dimensions=[time.dimensions[0]] +
list(grid.dimensions))
data.get_value()[0, :] = 1
data.get_value()[1, :] = 2
field = Field(grid=grid, time=time, is_data=data)
calc = [{'func': 'sum', 'name': 'sum'}]
ops = OcgOperations(dataset=field,
calc=calc,
calc_grouping='day',
calc_raw=True,
aggregate=True)
ret = ops.execute()
actual = ret.get_element(
variable_name='sum').get_masked_value().flatten()
self.assertNumpyAll(actual, np.ma.array([12.0, 24.0]))
