def test_system_spatial_averaging_through_operations(self):
data_name = 'data'
with vm.scoped('write', [0]):
if not vm.is_null:
x = Variable('x', range(5), 'x', float)
y = Variable('y', range(7), 'y', float)
grid = Grid(x, y)
data_value = np.arange(x.size * y.size).reshape(grid.shape)
data = Variable(data_name, data_value, grid.dimensions, float)
data_value = data.get_value()
field = Field(grid=grid, is_data=data)
path = self.get_temporary_file_path('data.nc')
field.write(path)
else:
data_value, path = None, None
data_value = MPI_COMM.bcast(data_value)
path = MPI_COMM.bcast(path)
rd = RequestDataset(path, variable=data_name)
ops = OcgOperations(dataset=rd, aggregate=True)
ret = ops.execute()
if ret is None:
self.assertNotEqual(vm.rank, vm.root)
else:
out_field = ret.get_element()
if MPI_RANK == 0:
desired = data_value.mean()
actual = out_field.data_variables[0].get_value()[0]
self.assertEqual(actual, desired)
def get_ocgis_field_from_esmf_field(efield, field=None):
"""
:param efield: The ESMPy field object to convert to an OCGIS field.
:type efield: :class:`ESMF.Field`
:param field: If provided, use this as the template field for OCGIS field creation.
:type field: :class:`~ocgis.Field`
:return: :class:`~ocgis.Field`
"""
ometa = efield._ocgis
dimnames = ometa.get('dimnames')
dimnames_backref = ometa.get('dimnames_backref')
ogrid = ometa.get('ocgis_grid')
if ogrid is None:
ogrid = get_ocgis_grid_from_esmf_grid(efield.grid)
ovar = None
if dimnames is not None and efield.name is not None:
ovar = Variable(name=efield.name, value=efield.data, dimensions=dimnames, dtype=efield.data.dtype)
broadcast_variable(ovar, get_dimension_names(dimnames_backref))
ovar.set_dimensions(dimnames_backref, force=True)
if field is None:
field = Field(grid=ogrid)
else:
field.set_grid(ogrid)
if ovar is not None:
field.add_variable(ovar, is_data=True, force=True)
if ogrid.has_mask:
field.grid.set_mask(ogrid.get_mask(), cascade=True)
return field
def test_system_dataset_identifiers_on_variables(self):
"""Test dataset identifiers make it to output variables for iteration."""
paths = []
variables = []
for suffix in [1, 2]:
path = self.get_temporary_file_path('foo{}.nc'.format(suffix))
paths.append(path)
x = Variable(name='x{}'.format(suffix), value=[2, 3], dimensions='x')
y = Variable(name='y{}'.format(suffix), value=[4, 5, 6], dimensions='y')
data_variable_name = 'data{}'.format(suffix)
variables.append(data_variable_name)
data = Variable(name=data_variable_name, value=np.arange(6).reshape(2, 3) + suffix,
dimensions=['x', 'y'])
grid = Grid(x, y)
field = Field(grid=grid, is_data=data)
field.write(path)
rds = [RequestDataset(uri=p, variable=dv) for p, dv in zip(paths, variables)]
ops = OcgOperations(dataset=rds)
rds_uids = [ds.uid for ds in ops.dataset]
self.assertEqual(rds_uids, [1, 2])
ret = ops.execute()
for field in ret.iter_fields():
self.assertFalse(field.grid.has_allocated_abstraction_geometry)
for variable in list(field.values()):
if isinstance(variable, CoordinateReferenceSystem):
continue
self.assertIsNotNone(variable._request_dataset.uid)
for row in variable.get_iter():
self.assertIsNotNone(row[HeaderName.DATASET_IDENTIFER])
def test_system_get_field_dimensioned_variables(self):
"""Test data is appropriately tagged to identify dimensioned variables."""
path = self.get_temporary_file_path('foo.nc')
time = TemporalVariable(value=[1, 2, 3], dimensions='time')
x = Variable(name='x', value=[10, 20], dimensions='x')
y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y')
data1 = Variable(name='data1',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data2 = Variable(name='data2',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time'])
field = Field(time=time,
grid=Grid(x, y),
variables=[data1, data2, data3])
field.write(path)
# Test dimensioned variables are read from a file with appropriate metadata.
rd = RequestDataset(path)
self.assertEqual(rd.variable, ('data1', 'data2'))
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data1', 'data2'))
# Test dimensioned variables are overloaded.
rd = RequestDataset(path, variable='data2')
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data2', ))
def create_raw_field(self, group_metadata=None, group_name=None, name=None, source_name=constants.UNINITIALIZED,
parent=None, uid=None):
"""
Create a raw field object. This field object should interpret metadata explicitly (i.e. no dimension map). In
general this method should not be overloaded by subclasses.
:param dict group_metadata: Metadata dictionary for the current group.
:param str group_name: The name of the current group being processed.
:param str name: See :class:`~ocgis.base.AbstractNamedObject`
:param str source_name: See :class:`~ocgis.base.AbstractNamedObject`
:param parent: :class:`~ocgis.variable.base.AbstractContainer`
:param int uid: See :class:`~ocgis.variable.base.AbstractContainer`
:return: :class:`~ocgis.Field`
"""
if group_metadata is None:
group_metadata = self.rd.metadata
field = Field(name=name, source_name=source_name, uid=uid, attrs=group_metadata.get('global_attributes'))
for var in self.create_variables(group_metadata, parent=field).values():
field.add_variable(var, force=True)
if parent is not None:
parent.add_child(field)
for k, v in group_metadata.get('groups', {}).items():
_ = self.create_raw_field(v, name=k, parent=field, group_name=k)
return field
def test_system_create_field_dimensioned_variables(self):
"""Test data is appropriately tagged to identify dimensioned variables."""
path = self.get_temporary_file_path('foo.nc')
time = TemporalVariable(value=[1, 2, 3], dimensions='time')
x = Variable(name='x', value=[10, 20], dimensions='x')
y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y')
data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x'])
data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x'])
data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time'])
field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3])
field.write(path)
# Test dimensioned variables are read from a file with appropriate metadata.
rd = RequestDataset(path)
self.assertEqual(rd.variable, ('data1', 'data2'))
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data1', 'data2'))
# Test dimensioned variables are overloaded.
rd = RequestDataset(path, variable='data2')
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data2',))
def test_grid(self):
# Test mask variable information is propagated through property.
grid = self.get_gridxy(with_xy_bounds=True)
self.assertTrue(grid.is_vectorized)
self.assertTrue(grid.has_bounds)
np.random.seed(1)
value = np.random.rand(*grid.shape)
select = value > 0.4
mask_var = create_spatial_mask_variable('nonstandard', select, grid.dimensions)
grid.set_mask(mask_var)
field = Field(grid=grid)
self.assertTrue(field.grid.has_bounds)
self.assertEqual(field.dimension_map.get_spatial_mask(), mask_var.name)
self.assertNumpyAll(field.grid.get_mask(), mask_var.get_mask())
# Test dimension map bounds are updated appropriately.
dim = Dimension('count', 2)
x = Variable(name='x', value=[1., 2.], dimensions=dim)
y = Variable(name='y', value=[1., 2.], dimensions=dim)
xb = Variable(name='xb', value=[[0., 1.5], [1.5, 2.5]], dimensions=[dim, 'bounds'])
yb = Variable(name='yb', value=[[0., 1.5], [1.5, 2.5]], dimensions=[dim, 'bounds'])
variables = [x, y, xb, yb]
dmap = DimensionMap()
dmap.set_variable(DMK.X, x, bounds=xb)
dmap.set_variable(DMK.Y, y, bounds=yb)
f = Field(dimension_map=dmap, variables=variables)
self.assertTrue(f.grid.has_bounds)
def create_raw_field(self,
group_metadata=None,
group_name=None,
name=None,
source_name=constants.UNINITIALIZED,
parent=None,
uid=None):
"""
Create a raw field object. This field object should interpret metadata explicitly (i.e. no dimension map). In
general this method should not be overloaded by subclasses.
:param dict group_metadata: Metadata dictionary for the current group.
:param str group_name: The name of the current group being processed.
:param str name: See :class:`~ocgis.base.AbstractNamedObject`
:param str source_name: See :class:`~ocgis.base.AbstractNamedObject`
:param parent: :class:`~ocgis.variable.base.AbstractContainer`
:param int uid: See :class:`~ocgis.variable.base.AbstractContainer`
:return: :class:`~ocgis.Field`
"""
if group_metadata is None:
group_metadata = self.rd.metadata
field = Field(name=name,
source_name=source_name,
uid=uid,
attrs=group_metadata.get('global_attributes'))
for var in self.create_variables(group_metadata,
parent=field).values():
field.add_variable(var, force=True)
if parent is not None:
parent.add_child(field)
for k, v in group_metadata.get('groups', {}).items():
_ = self.create_raw_field(v, name=k, parent=field, group_name=k)
return field
def test_set_geom(self):
f = Field()
self.assertIsNone(f.crs)
g = GeometryVariable(value=[Point(1, 2)], dimensions='geom', crs=Spherical())
f.set_geom(g)
def test_init(self):
field = self.get_ocgfield()
self.assertIsInstance(field, Field)
# Test unique identifier.
field = Field(uid=4)
self.assertEqual(field.uid, 4)
# Test with a coordinate system and geometry.
desired_crs = WGS84()
geom = GeometryVariable(name='geom',
value=[Point(1, 2)],
dimensions='geom')
field = Field(crs=desired_crs, geom=geom)
self.assertEqual(field.crs, desired_crs)
# Test dimension names are automatically added to dimension map.
g = GeometryVariable(name='geom',
value=[Point(1, 2)],
dimensions='the_geom_dim')
f = Field(geom=g)
actual = f.dimension_map.get_dimension(DimensionMapKey.GEOM)
self.assertEqual(actual, ['the_geom_dim'])
# Test dimension map does not have any entries at initialization.
actual = Field()
desired = actual.dimension_map.as_dict()
self.assertEqual(len(desired), 0)
def test_system_with_time_data(self):
"""Test writing data with a time dimension."""
path = self.get_temporary_file_path('what.shp')
t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time')
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)],
name='geom',
dimensions='time')
field = Field(variables=[t, geom],
dimension_map={
'time': {
'variable': 'time'
},
'geom': {
'variable': 'geom'
}
})
field.write(path,
iter_kwargs={'variable': 'time'},
driver=DriverVector)
rd = RequestDataset(uri=path)
field2 = rd.get()
# netcdftime worthlessness
poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'],
['1-01-02 12:00:00', '1-01-03 12:00:00']]
actual = field2['TIME'].get_value().tolist()
res = [p == actual for p in poss]
self.assertTrue(any(res))
def test_get_by_tag(self):
v1 = Variable(name='tas')
v2 = Variable(name='tasmax')
v3 = Variable(name='tasmin')
tags = {'avg': ['tas'], 'other': ['tasmax', 'tasmin']}
field = Field(variables=[v1, v2, v3], tags=tags)
t = field.get_by_tag('other')
self.assertAsSetEqual([ii.name for ii in t], tags['other'])
def test_get_by_tag(self):
v1 = Variable(name='tas')
v2 = Variable(name='tasmax')
v3 = Variable(name='tasmin')
tags = {'avg': ['tas'], 'other': ['tasmax', 'tasmin']}
field = Field(variables=[v1, v2, v3], tags=tags)
t = field.get_by_tag('other')
self.assertAsSetEqual([ii.name for ii in t], tags['other'])
def test_write_variable_collection_different_data_types(self):
"""Test multiple data types are handled by the shapefile write when melted is True."""
v_int = Variable(name='an_int', value=[1, 2, 3], dtype=int, dimensions='three')
v_flt = Variable(name='a_float', value=[10., 20., 30.], dtype=float, dimensions='three')
g = GeometryVariable(name='points', value=[Point(1, 2), Point(3, 4), Point(5, 6)], dimensions='three')
field = Field(is_data=[v_int, v_flt], geom=g)
self.assertEqual(len(field.data_variables), 2)
path = self.get_temporary_file_path('foo.shp')
field.write(path, driver='vector', iter_kwargs=dict(melted=True))
def test_write_variable_collection_different_data_types(self):
"""Test multiple data types are handled by the shapefile write when melted is True."""
v_int = Variable(name='an_int', value=[1, 2, 3], dtype=int, dimensions='three')
v_flt = Variable(name='a_float', value=[10., 20., 30.], dtype=float, dimensions='three')
g = GeometryVariable(name='points', value=[Point(1, 2), Point(3, 4), Point(5, 6)], dimensions='three')
field = Field(is_data=[v_int, v_flt], geom=g)
self.assertEqual(len(field.data_variables), 2)
path = self.get_temporary_file_path('foo.shp')
field.write(path, driver='vector', iter_kwargs=dict(melted=True))
def test_iter_masking_and_driver(self):
"""Test mask is set to None."""
time = TemporalVariable(value=[3, 4, 5], dimensions='time')
data = Variable(value=[7, 8, 9], name='data', mask=[False, True, False], dimensions='time')
field = Field(time=time, is_data=data, variables=data)
itr = field.iter(allow_masked=True)
actual = list(itr)
self.assertIsNone(actual[1][1][data.name])
def test_system_conform_units(self):
"""Test conforming units on data read from shapefile."""
path = self.get_temporary_file_path('temps.shp')
gvar = GeometryVariable(value=[Point(1, 2), Point(3, 4)], dimensions='g', name='geom')
var = Variable(name='temp', value=[10., 20.], dimensions='g')
field = Field(variables=[gvar, var], geom=gvar, is_data=var)
field.write(path, driver=DriverVector)
field = RequestDataset(path, units='celsius', variable='temp', conform_units_to='fahrenheit').get()
self.assertNumpyAllClose(field['temp'].get_value(), np.array([50., 68.]))
def test_system_conform_units(self):
"""Test conforming units on data read from shapefile."""
path = self.get_temporary_file_path('temps.shp')
gvar = GeometryVariable(value=[Point(1, 2), Point(3, 4)], dimensions='g', name='geom')
var = Variable(name='temp', value=[10., 20.], dimensions='g')
field = Field(variables=[gvar, var], geom=gvar, is_data=var)
field.write(path, driver=DriverVector)
field = RequestDataset(path, units='celsius', variable='temp', conform_units_to='fahrenheit').get()
self.assertNumpyAllClose(field['temp'].get_value(), np.array([50., 68.]))
def test_update_crs(self):
# Test copying allows the CRS to be updated on the copy w/out changing the source CRS.
desired = Spherical()
gvar = GeometryVariable(value=[Point(1, 2)],
name='geom',
dimensions='geom')
field = Field(crs=desired, geom=gvar)
cfield = field.copy()
self.assertEqual(cfield.crs, desired)
new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326)
cfield.update_crs(new_crs)
self.assertEqual(field.crs, desired)
def test_write_variable_collection(self):
# Attempt to write without a geometry variable.
v = Variable('a', value=[1, 2], dimensions='bb')
field = Field(variables=v)
path = self.get_temporary_file_path('out.shp')
with self.assertRaises(ValueError):
field.write(path, driver=DriverVector)
# Test writing a field with two-dimensional geometry storage.
value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)]
gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms')
gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)])
var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a'])
var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat'])
var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon'])
var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat'])
field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data'])
path = self.get_temporary_file_path('2d.shp')
field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertTrue(len(read) > 2)
self.assertEqual(list(read.keys()),
['data', 'some_lats', 'some_lons', constants.VariableName.GEOMETRY_VARIABLE])
# Test writing a subset of the variables.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1])
field.write(path, variable_names=['keep'], driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertNotIn('remove', read)
# Test using append.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2])
for idx in range(3):
sub = field[{'points': idx}]
if idx == 0:
write_mode = MPIWriteMode.WRITE
else:
write_mode = MPIWriteMode.APPEND
sub.write(path, write_mode=write_mode, driver=DriverVector)
self.assertOGRFileLength(path, idx + 1)
def parse(self, value):
if type(value) in [list, tuple]:
if all([isinstance(element, dict) for element in value]):
for ii, element in enumerate(value, start=1):
if 'geom' not in element:
ocgis_lh(exc=DefinitionValidationError(self, 'Geometry dictionaries must have a "geom" key.'))
if 'properties' not in element:
element['properties'] = {self._ugid_key: ii}
crs = element.get('crs', constants.UNINITIALIZED)
if 'crs' not in element:
ocgis_lh(msg='No CRS in geometry dictionary - assuming WGS84.', level=logging.WARN)
ret = Field.from_records(value, crs=crs, uid=self.geom_uid, union=self.union,
data_model=self.data_model)
else:
if len(value) == 2:
geom = Point(value[0], value[1])
elif len(value) == 4:
minx, miny, maxx, maxy = value
geom = Polygon(((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny)))
if not geom.is_valid:
raise DefinitionValidationError(self, 'Parsed geometry is not valid.')
ret = [{'geom': geom, 'properties': {self._ugid_key: 1}}]
ret = Field.from_records(ret, uid=self.geom_uid, union=self.union, data_model=self.data_model)
self._bounds = geom.bounds
elif isinstance(value, GeomCabinetIterator):
self._shp_key = value.key or value.path
# Always yield fields.
value.as_field = True
ret = value
elif isinstance(value, BaseGeometry):
ret = [{'geom': value, 'properties': {self._ugid_key: 1}}]
ret = Field.from_records(ret, uid=self.geom_uid, union=self.union, data_model=self.data_model)
elif value is None:
ret = value
elif isinstance(value, Field):
ret = value
elif isinstance(value, GeometryVariable):
if value.ugid is None:
msg = 'Geometry variables must have an associated "UGID".'
raise DefinitionValidationError(self, msg)
ret = Field(geom=value, crs=value.crs)
else:
raise NotImplementedError(type(value))
# Convert to singular field if this is a field object.
if isinstance(ret, Field):
ret = tuple(self._iter_singular_fields_(ret))
return ret
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_get_dimension_map_2d_spatial_coordinates(self):
grid = create_gridxy_global()
grid.expand()
path = self.get_temporary_file_path('foo.nc')
f = Field(grid=grid)
f.write(path)
rd = RequestDataset(path)
field = rd.get()
sub = field.get_field_slice({'y': 10, 'x': 5})
self.assertEqual(sub.grid.x.shape, (1, 1))
actual = f.dimension_map.get_dimension(DimensionMapKey.Y)
self.assertEqual(actual, ['y'])
actual = f.dimension_map.get_dimension(DimensionMapKey.X)
self.assertEqual(actual, ['x'])
def get_field(self, ntime=2, variable_name='foo', nrow=2, ncol=2):
"""Create random field where mean varies with radius and std with the
angle around the center of the grid.
"""
np.random.seed(1)
row = Variable(value=np.arange(nrow) - nrow / 2., name='row', dimensions='row')
col = Variable(value=np.arange(ncol) - ncol / 2., name='col', dimensions='col')
grid = Grid(col, row)
x, y = grid.get_value_stacked()
start = dt.datetime(2000, 1, 1)
delta = dt.timedelta(days=1)
value_temporal = [start + i * delta for i in range(ntime)]
temporal = TemporalVariable(value=value_temporal, dimensions='time',
name='time')
nlevel = 1
level = None
nrlz = 1
realization = None
value = np.random.rand(nrlz, ntime, nlevel, nrow, ncol) * np.arctan2(x, y).clip(.1) + np.hypot(x, y)
variable = Variable(name=variable_name,
value=value,
dimensions=['realization', 'time', 'level', 'row',
'col'])
field = Field(grid=grid, time=temporal, is_data=variable, level=level,
realization=realization)
return field
def test_from_records(self):
gci = GeomCabinetIterator(path=self.path_state_boundaries)
actual = Field.from_records(gci, data_model='NETCDF3_CLASSIC')
desired = {'UGID': np.int32,
'ID': np.int32}
for v in desired.keys():
self.assertEqual(actual[v].get_value().dtype, desired[v])
def test_time(self):
units = [None, 'days since 2012-1-1']
calendar = [None, '365_day']
value = [10, 20]
bounds = [[5, 15], [15, 25]]
variable_type = [Variable, TemporalVariable]
bounds_variable_type = [Variable, TemporalVariable]
keywords = dict(units=units, calendar=calendar, variable_type=variable_type,
bounds_variable_type=bounds_variable_type)
for k in self.iter_product_keywords(keywords):
attrs = {'units': k.units, 'calendar': k.calendar}
dimension_map = {'time': {'variable': 'time', 'bounds': 'time_bnds', 'attrs': attrs}}
var = k.variable_type(name='time', value=value, attrs=attrs, dimensions=['one'])
bounds_var = k.bounds_variable_type(name='time_bnds', value=bounds, dimensions=['one', 'two'])
f = Field(variables=[var, bounds_var], dimension_map=dimension_map)
self.assertTrue(len(f.dimension_map._storage) == 1)
self.assertTrue(f.time.has_bounds)
self.assertIsInstance(f.time, TemporalVariable)
self.assertIsInstance(f.time.bounds, TemporalVariable)
self.assertEqual(f.time.value_datetime.shape, (2,))
self.assertEqual(f.time.bounds.value_datetime.shape, (2, 2))
if k.units is None:
desired = constants.DEFAULT_TEMPORAL_UNITS
else:
desired = k.units
self.assertEqual(f.time.units, desired)
if k.calendar is None:
desired = constants.DEFAULT_TEMPORAL_CALENDAR
else:
desired = k.calendar
self.assertEqual(f.time.calendar, desired)
self.assertEqual(f.time.bounds.calendar, desired)
def get_wrap_field(crs=None, unwrapped=True):
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=False)
ompi.create_dimension('y', 7, dist=True)
ompi.create_dimension('time', size_current=4, dist=False)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
row = Variable(value=[-60, -40, -20, 0, 20, 40, 60], name='y', dimensions='y')
if unwrapped:
col_value = [1, 90, 180, 225, 270]
else:
col_value = [-170, -85, 0, 85, 170]
col = Variable(value=col_value, name='x', dimensions='x')
grid = Grid(col, row)
value = np.zeros((4, 7, 5))
for col_idx in range(value.shape[-1]):
value[:, :, col_idx] = col_idx
time = TemporalVariable(name='time', value=[1, 2, 3, 4], dimensions='time')
var = Variable(name='foo', value=value, dimensions=['time', 'y', 'x'])
field = Field(grid=grid, is_data=var, crs=crs, time=time)
else:
field = None
field = variable_collection_scatter(field, ompi)
return field
def test_system_process_geometries(self):
"""Test multiple geometries with coordinate system update."""
a = 'POLYGON((-105.21347987288135073 40.21514830508475313,-104.39928495762711691 40.21514830508475313,-104.3192002118643984 39.5677966101694949,-102.37047139830508513 39.61451271186440692,-102.12354343220337682 37.51896186440677639,-105.16009004237288593 37.51896186440677639,-105.21347987288135073 40.21514830508475313))'
b = 'POLYGON((-104.15235699152542281 39.02722457627118757,-103.71189088983049942 39.44099576271186436,-102.71750529661017026 39.28082627118644155,-102.35712394067796538 37.63908898305084705,-104.13900953389830306 37.63241525423728717,-104.15235699152542281 39.02722457627118757))'
geom = [{'geom': wkt.loads(xx), 'properties': {'UGID': ugid}} for ugid, xx in enumerate([a, b])]
grid_value = [
[[37.0, 37.0, 37.0, 37.0], [38.0, 38.0, 38.0, 38.0], [39.0, 39.0, 39.0, 39.0], [40.0, 40.0, 40.0, 40.0]],
[[-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0],
[-105.0, -104.0, -103.0, -102.0]]]
output_crs = CoordinateReferenceSystem(
value={'a': 6370997, 'lon_0': -100, 'y_0': 0, 'no_defs': True, 'proj': 'laea', 'x_0': 0, 'units': 'm',
'b': 6370997, 'lat_0': 45})
x = Variable('x', grid_value[1], dimensions=['lat', 'lon'])
y = Variable('y', grid_value[0], dimensions=['lat', 'lon'])
grid = Grid(x, y)
field = Field(grid=grid, crs=Spherical())
ops = OcgOperations(dataset=field, geom=geom, output_crs=output_crs)
ret = ops.execute()
expected = {0: -502052.79407259845,
1: -510391.37909706926}
for field, container in ret.iter_fields(yield_container=True):
self.assertAlmostEqual(field.grid.get_value_stacked().mean(),
expected[container.geom.ugid.get_value()[0]])
def test_system_regridding_crs(self):
"""Test with coordinate systems."""
dest_crs = WGS84()
grid_spherical = self.get_gridxy_global(resolution=10.0, wrapped=False, crs=Spherical())
self.assertEqual(grid_spherical.crs, Spherical())
coords = grid_spherical.get_value_stacked()
data_value = self.get_exact_field_value(coords[1], coords[0])
desired = data_value.copy()
data_var = Variable(name='data_src', value=data_value, dimensions=grid_spherical.dimensions)
source = Field(grid=grid_spherical, is_data=data_var, crs=grid_spherical.crs)
self.assertEqual(source.crs, Spherical())
destination = deepcopy(source)
destination.update_crs(dest_crs)
source_expanded = deepcopy(source.grid)
source_expanded.expand()
diff = np.abs(destination.y.get_value() - source_expanded.y.get_value())
self.assertAlmostEqual(diff.max(), 0.19231511439)
for output_crs in [None, WGS84()]:
ops = OcgOperations(dataset=source, regrid_destination=destination, output_crs=output_crs)
ret = ops.execute()
actual = ret.get_element(variable_name=data_var.name)
if output_crs is None:
self.assertEqual(actual.parent.crs, Spherical())
else:
self.assertEqual(actual.parent.crs, WGS84())
actual = actual.get_value()
diff = np.abs(actual - desired)
self.assertTrue(diff.max() < 1e-5)
def add_field(self, field, container, force=False):
"""
Add a field to the spatial collection.
:param field: The field to add.
:type field: :class:`~ocgis.Field`
:param container: The container geometry. A ``None`` value is allowed.
:type container: :class:`~ocgis.Field` | ``None``
:param bool force: If ``True``, clobber any field names in the spatial collection.
:return:
"""
# Assume a NoneType container if there is no geometry associated with the container.
if container is not None and container.geom is not None:
ugid = container.geom.ugid.get_value()[0]
if ugid not in self.children:
self.children[ugid] = container
else:
# We want to use the reference to the container in the collection.
container = self.children[ugid]
container.add_child(field, force=force)
else:
if None not in self.children:
self.children[None] = Field()
container = self.children[None]
container.add_child(field, force=force)
def test_create_dimension_map_2d_spatial_coordinates(self):
grid = create_gridxy_global()
grid.expand()
path = self.get_temporary_file_path('foo.nc')
f = Field(grid=grid)
f.write(path)
rd = RequestDataset(path)
field = rd.get()
sub = field.get_field_slice({'y': 10, 'x': 5})
self.assertEqual(sub.grid.x.shape, (1, 1))
actual = f.dimension_map.get_dimension(DimensionMapKey.Y)
self.assertEqual(actual, ['y'])
actual = f.dimension_map.get_dimension(DimensionMapKey.X)
self.assertEqual(actual, ['x'])
def test_properties(self):
# Test an empty properties dictionary is created if there are no data variables on the container/parent field.
gvar = GeometryVariable(name='empty', value=[Point(1, 2)], ugid=20, dimensions='ngeom')
info = Variable(name='info')
gvar.parent.add_variable(info)
sc = SpatialCollection()
sc.add_field(Field(name='what'), gvar.parent)
self.assertEqual(len(sc.properties), 1)
def test_system_dimension_map_formatting(self):
"""Test any formatting of the incoming dimension map by the field."""
dmap = {'time': {'variable': 'time'}}
time = TemporalVariable(name='time', value=[1, 2, 3], dimensions='the_time')
field = Field(time=time, dimension_map=dmap)
actual = field.dimension_map.get_dimension('time')
self.assertEqual(actual, ['the_time'])
def test_crs(self):
"""Test overloading by geometry and grid."""
field = Field()
self.assertIsNone(field.crs)
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical())
field = Field(geom=geom)
self.assertEqual(field.crs, geom.crs)
grid = self.get_gridxy_global(crs=Spherical())
field = Field(grid=grid)
self.assertEqual(field.crs, grid.crs)
grid = self.get_gridxy_global(crs=Spherical())
# Grid and field coordinate systems do not match.
with self.assertRaises(ValueError):
Field(grid=grid, crs=WGS84())
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical())
with self.assertRaises(ValueError):
Field(geom=geom, crs=WGS84())
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g')
grid = self.get_gridxy_global()
field = Field(geom=geom, grid=grid, crs=WGS84())
self.assertEqual(field.crs, WGS84())
self.assertEqual(field.geom.crs, WGS84())
self.assertEqual(field.grid.crs, WGS84())
g = self.get_gridxy_global()
f = Field(grid=g, crs=Spherical())
self.assertIn('standard_name', f.grid.x.attrs)
self.assertIn('standard_name', f.grid.y.attrs)
def test_update_crs(self):
# Test copying allows the CRS to be updated on the copy w/out changing the source CRS.
desired = Spherical()
gvar = GeometryVariable(value=[Point(1, 2)], name='geom', dimensions='geom')
field = Field(crs=desired, geom=gvar)
cfield = field.copy()
self.assertEqual(cfield.crs, desired)
new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326)
cfield.update_crs(new_crs)
self.assertEqual(field.crs, desired)
# Test geometry crs update is called.
mfield = Mock(Field)
mfield.is_empty = False
mfield.dimension_map = Mock(DimensionMap)
mfield.grid = Mock(Grid)
mfield.geom = Mock(GeometryVariable)
mcrs = Mock(Spherical)
Field.update_crs(mfield, mcrs)
mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs)
mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs)
from_crs = Mock(WGS84)
mfield.grid.update_crs.reset_mock()
mfield.geom.update_crs.reset_mock()
Field.update_crs(mfield, mcrs, from_crs=from_crs)
mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
def test_system_with_time_data(self):
"""Test writing data with a time dimension."""
path = self.get_temporary_file_path('what.shp')
t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time')
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='time')
field = Field(variables=[t, geom], dimension_map={'time': {'variable': 'time'},
'geom': {'variable': 'geom'}})
field.write(path, iter_kwargs={'variable': 'time'}, driver=DriverVector)
rd = RequestDataset(uri=path)
field2 = rd.get()
# netcdftime worthlessness
poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'], ['1-01-02 12:00:00', '1-01-03 12:00:00']]
actual = field2['TIME'].get_value().tolist()
res = [p == actual for p in poss]
self.assertTrue(any(res))
def test_system_masking(self):
"""Test how the mask is handled by the NetCDF python library."""
var = Variable(name='foo', value=[1, 2, 3], dimensions='three')
self.assertIsNone(var.get_mask())
path = self.get_temporary_file_path('foo.nc')
var.parent.write(path)
invar = Field.read(path)['foo']
self.assertIsNone(invar.get_mask())
def create_regridding_field(grid, name_variable):
col_shape = grid.shape[1]
row_shape = grid.shape[0]
value = np.ones(col_shape * row_shape, dtype=float).reshape(
grid.shape) * 15.
variable = Variable(name=name_variable,
value=value,
dimensions=grid.dimensions)
field = Field(is_data=variable, grid=grid, crs=Spherical())
return field
def get_field(self):
gridxy = self.get_gridxy(with_xy_bounds=True)
coords_stacked = gridxy.get_value_stacked()
exact = self.get_exact_field_value(coords_stacked[1], coords_stacked[0])
exact = Variable(name='exact1', value=exact, dimensions=gridxy.dimensions)
dimension_map = {'x': {'variable': 'x', 'bounds': 'xbounds'}, 'y': {'variable': 'y', 'bounds': 'ybounds'}}
field1 = Field.from_variable_collection(gridxy.parent, dimension_map=dimension_map)
field1.add_variable(exact)
field1.set_name('exact1')
field1.append_to_tags(TagName.DATA_VARIABLES, 'exact1')
return field1
def test_set_x(self):
f = Field()
var = Variable('x', value=[1, 2], dimensions='xdim')
f.set_x(var, 'xdim')
var2 = Variable('x2', value=[3, 4], dimensions='xdim2')
f.set_x(var2, 'xdim2')
self.assertNotIn(var.name, f)
f.set_x(None, None)
self.assertEqual(len(f), 0)
self.assertIsNone(f.x)
def get_field(self):
gridxy = self.get_gridxy(with_xy_bounds=True)
coords_stacked = gridxy.get_value_stacked()
exact = self.get_exact_field_value(coords_stacked[1], coords_stacked[0])
exact = Variable(name='exact1', value=exact, dimensions=gridxy.dimensions)
dimension_map = {'x': {'variable': 'x', 'bounds': 'xbounds'}, 'y': {'variable': 'y', 'bounds': 'ybounds'}}
field1 = Field.from_variable_collection(gridxy.parent, dimension_map=dimension_map)
field1.add_variable(exact)
field1.set_name('exact1')
field1.append_to_tags(TagName.DATA_VARIABLES, 'exact1')
return field1
def get_subset_field(self):
crs = self.crs
geoms = self.geoms
gridcode = Variable('gridcode', [110101, 12103], dimensions='ngeom')
description = Variable('description', ['high point', 'low point'], dimensions='ngeom')
dimension_map = {'geom': {'variable': 'geoms', DimensionMapKey.DIMENSION: ['ngeom']},
'crs': {'variable': crs.name}}
poi = Field(variables=[geoms, gridcode, description], dimension_map=dimension_map,
is_data=[gridcode, description])
geoms.set_ugid(gridcode)
return poi
def _run_():
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure(to_stream=True)
records = [{
'geom': Point(1, 2),
'properties': {
'a_list': [1, 2, 3]
}
}]
actual = Field.from_records(records)
self.assertNotIn("a_list", actual.keys())
env.SUPPRESS_WARNINGS = True
def read_from_collection(target, request_dataset, parent=None, name=None, source_name=constants.UNINITIALIZED,
uid=None):
# Allow an empty variable renaming map. This occurs when there are no visible data variables to the metadata
# parser.
try:
rename_variable_map = request_dataset.rename_variable_map
except NoDataVariablesFound:
rename_variable_map = {}
ret = Field(attrs=get_netcdf_attributes(target), parent=parent, name=name, source_name=source_name, uid=uid)
pred = request_dataset.predicate
for varname, ncvar in target.variables.items():
if pred is not None and not pred(varname):
continue
source_name = varname
name = rename_variable_map.get(varname, varname)
sv = SourcedVariable(name=name, request_dataset=request_dataset, parent=ret, source_name=source_name)
ret[name] = sv
for group_name, ncgroup in list(target.groups.items()):
child = read_from_collection(ncgroup, request_dataset, parent=ret, name=group_name, uid=uid)
ret.add_child(child)
return ret
def test_write_parallel(self):
"""Test writing by selective rank."""
if MPI_SIZE != 3 and MPI_SIZE != 1:
raise SkipTest('MPI_SIZE != 1 or 3')
ranks = list(range(MPI_SIZE))
for base_rank in ranks:
for driver in [
DriverCSV,
DriverVector,
DriverNetcdf
]:
if MPI_RANK == 0:
path = self.get_temporary_file_path('{}-{}.{}'.format(driver.key, base_rank,
driver.common_extension))
else:
path = None
path = MPI_COMM.bcast(path)
with vm.scoped('field write by rank', [base_rank]):
if not vm.is_null:
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='geom')
data = Variable(name='data', value=[10, 20], dimensions='geom')
field = Field(geom=geom)
field.add_variable(data, is_data=True)
self.assertFalse(os.path.isdir(path))
field.write(path, driver=driver)
self.assertFalse(os.path.isdir(path))
rd = RequestDataset(path, driver=driver)
in_field = rd.get()
self.assertEqual(in_field['data'].dimensions[0].size, 2)
MPI_COMM.Barrier()
MPI_COMM.Barrier()
def test_system_crs_and_grid_abstraction(self):
f = Field(grid_abstraction='point')
grid = self.get_gridxy(with_xy_bounds=True)
f.add_variable(grid.x)
crs = CoordinateReferenceSystem(epsg=2136, name='location')
f.add_variable(crs)
self.assertIsNone(f.crs)
f.dimension_map.set_crs(crs)
f.dimension_map.set_variable('x', grid.x)
f.dimension_map.set_variable('y', grid.y)
self.assertEqual(f.grid.crs, crs)
f.set_geom(f.grid.get_abstraction_geometry())
self.assertEqual(f.grid.abstraction, 'point')
self.assertEqual(f.geom.geom_type, 'Point')
def test_get_field_write_target(self):
# Test coordinate system names are added to attributes of dimensioned variables.
x = Variable('x', dimensions='x', value=[1])
y = Variable('y', dimensions='y', value=[2])
t = Variable('t', dimensions='t', value=[3])
crs = WGS84()
d = Variable('data', dimensions=['t', 'y', 'x'], value=[[[1]]])
grid = Grid(x, y)
field = Field(grid=grid, time=t, crs=crs)
field.add_variable(d, is_data=True)
target = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(target[d.name].attrs['grid_mapping'], crs.name)
self.assertEqual(field.x.units, 'degrees_east')
# Test bounds units are removed when writing.
x = Variable(name='x', value=[1, 2, 3], dtype=float, dimensions='xdim', units='hours')
y = Variable(name='y', value=[1, 2, 3], dtype=float, dimensions='ydim', units='hours')
grid = Grid(x, y)
grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds')
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
field = Field(grid=grid)
actual = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(x.bounds.units, x.units)
self.assertNumpyMayShareMemory(actual[x.name].get_value(), field[x.name].get_value())
self.assertIsNone(actual[x.name].bounds.units)
self.assertIsNone(actual[y.name].bounds.units)
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
# Test actual coordinate system is triggered.
field = Field()
src = CFSpherical()
dst = WGS84()
field.set_crs(src)
self.assertEqual(field.crs, src)
self.assertIsNone(env.COORDSYS_ACTUAL)
env.COORDSYS_ACTUAL = dst
actual = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(actual.crs, dst)
self.assertEqual(field.crs, src)
self.assertNotIn(src.name, actual)
self.assertIn(dst.name, actual)
def _update_aggregation_wrapping_crs_(obj, alias, sfield, subset_sdim, subset_ugid):
raise_if_empty(sfield)
ocgis_lh('entering _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias,
ugid=subset_ugid, level=logging.DEBUG)
# Aggregate if requested.
if obj.ops.aggregate:
ocgis_lh('aggregate requested in _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias,
ugid=subset_ugid, level=logging.DEBUG)
# There may be no geometries if we are working with a gridded dataset. Load the geometries if this is the case.
sfield.set_abstraction_geom()
ocgis_lh('after sfield.set_abstraction_geom in _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias,
ugid=subset_ugid, level=logging.DEBUG)
# Union the geometries and spatially average the data variables.
# with vm.scoped(vm.get_live_ranks_from_object(sfield)):
sfield = sfield.geom.get_unioned(spatial_average=sfield.data_variables)
ocgis_lh('after sfield.geom.get_unioned in _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias,
ugid=subset_ugid, level=logging.DEBUG)
# None is returned for the non-root process. Check we are in parallel and create an empty field.
if sfield is None:
if vm.size == 1:
raise ValueError('None should not be returned from get_unioned if running on a single processor.')
else:
sfield = Field(is_empty=True)
else:
sfield = sfield.parent
vm.create_subcomm_by_emptyable(SubcommName.SPATIAL_AVERAGE, sfield, is_current=True, clobber=True)
if not vm.is_null and subset_sdim is not None and subset_sdim.geom is not None:
# Add the unique geometry identifier variable. This should match the selection geometry's identifier.
new_gid_variable_kwargs = dict(name=HeaderName.ID_GEOMETRY, value=subset_sdim.geom.ugid.get_value(),
dimensions=sfield.geom.dimensions)
dm = get_data_model(obj.ops)
new_gid_variable = create_typed_variable_from_data_model('int', data_model=dm, **new_gid_variable_kwargs)
sfield.geom.set_ugid(new_gid_variable)
if vm.is_null:
ocgis_lh(msg='null communicator following spatial average. returning.', logger=obj._subset_log,
level=logging.DEBUG)
return sfield
raise_if_empty(sfield)
ocgis_lh(msg='before wrapped_state in _update_aggregation_wrapping_crs_', logger=obj._subset_log,
level=logging.DEBUG)
try:
wrapped_state = sfield.wrapped_state
except WrappedStateEvalTargetMissing:
# If there is no target for wrapping evaluation, then consider this unknown.
wrapped_state = WrappedState.UNKNOWN
ocgis_lh(msg='after wrapped_state in _update_aggregation_wrapping_crs_', logger=obj._subset_log,
level=logging.DEBUG)
# Wrap the returned data.
if not env.OPTIMIZE_FOR_CALC and not sfield.is_empty:
if wrapped_state == WrappedState.UNWRAPPED:
ocgis_lh('wrap target is empty: {}'.format(sfield.is_empty), obj._subset_log, level=logging.DEBUG)
# There may be no geometries if we are working with a gridded dataset. Load the geometries if this
# is the case.
sfield.set_abstraction_geom()
if obj.ops.output_format in constants.VECTOR_OUTPUT_FORMATS and obj.ops.vector_wrap:
ocgis_lh('wrapping output geometries', obj._subset_log, alias=alias, ugid=subset_ugid,
level=logging.DEBUG)
# Deepcopy geometries before wrapping as wrapping will be performed inplace. The original field may
# need to be reused for additional subsets.
geom = sfield.geom
copied_geom = geom.get_value().copy()
geom.set_value(copied_geom)
# Some grids do not play nicely with wrapping. Bounds may be less than zero for an unwrapped grid.
# Force wrapping if it is requested. Normally, when force is false there is a pass-through that will
# leave the data untouched.
geom.wrap(force=True)
ocgis_lh('finished wrapping output geometries', obj._subset_log, alias=alias, ugid=subset_ugid,
level=logging.DEBUG)
# Transform back to rotated pole if necessary.
original_rotated_pole_crs = obj._backtransform.get(constants.BackTransform.ROTATED_POLE)
if original_rotated_pole_crs is not None:
if not isinstance(obj.ops.output_crs, (Spherical, WGS84)):
sfield.update_crs(original_rotated_pole_crs)
# Update the coordinate system of the data output.
if obj.ops.output_crs is not None:
# If the geometry is not none, it may need to be projected to match the output coordinate system.
if subset_sdim is not None and subset_sdim.crs != obj.ops.output_crs:
subset_sdim.update_crs(obj.ops.output_crs)
# Update the subsetted field's coordinate system.
sfield = sfield.copy()
sfield.update_crs(obj.ops.output_crs)
# Wrap or unwrap the data if the coordinate system permits.
_update_wrapping_(obj, sfield)
ocgis_lh('leaving _update_aggregation_wrapping_crs_', obj._subset_log, level=logging.DEBUG)
return sfield
def write_chunks(self):
"""
Write grid subsets to netCDF files using the provided filename templates. This will also generate ESMF
regridding weights for each subset if requested.
"""
src_filenames = []
dst_filenames = []
wgt_filenames = []
dst_slices = []
src_slices = []
index_path = self.create_full_path_from_template('index_file')
# nzeros = len(str(reduce(lambda x, y: x * y, self.nchunks_dst)))
ctr = 1
ocgis_lh(logger='grid_chunker', msg='starting self.iter_src_grid_subsets', level=logging.DEBUG)
for sub_src, src_slc, sub_dst, dst_slc in self.iter_src_grid_subsets(yield_dst=True):
ocgis_lh(logger='grid_chunker', msg='finished iteration {} for self.iter_src_grid_subsets'.format(ctr),
level=logging.DEBUG)
src_path = self.create_full_path_from_template('src_template', index=ctr)
dst_path = self.create_full_path_from_template('dst_template', index=ctr)
wgt_path = self.create_full_path_from_template('wgt_template', index=ctr)
src_filenames.append(os.path.split(src_path)[1])
dst_filenames.append(os.path.split(dst_path)[1])
wgt_filenames.append(wgt_path)
dst_slices.append(dst_slc)
src_slices.append(src_slc)
# Only write destinations if an iterator is not provided.
if self.iter_dst is None:
zip_args = [[sub_src, sub_dst], [src_path, dst_path]]
else:
zip_args = [[sub_src], [src_path]]
cc = 1
for target, path in zip(*zip_args):
with vm.scoped_by_emptyable('field.write' + str(cc), target):
if not vm.is_null:
ocgis_lh(logger='grid_chunker', msg='write_chunks:writing: {}'.format(path),
level=logging.DEBUG)
field = Field(grid=target)
field.write(path)
ocgis_lh(logger='grid_chunker', msg='write_chunks:finished writing: {}'.format(path),
level=logging.DEBUG)
cc += 1
# Increment the counter outside of the loop to avoid counting empty subsets.
ctr += 1
# Generate an ESMF weights file if requested and at least one rank has data on it.
if self.genweights and len(vm.get_live_ranks_from_object(sub_src)) > 0:
vm.barrier()
self.write_esmf_weights(src_path, dst_path, wgt_path, src_grid=sub_src, dst_grid=sub_dst)
vm.barrier()
# Global shapes require a VM global scope to collect.
src_global_shape = global_grid_shape(self.src_grid)
dst_global_shape = global_grid_shape(self.dst_grid)
# Gather and collapse source slices as some may be empty and we write on rank 0.
gathered_src_grid_slice = vm.gather(src_slices)
if vm.rank == 0:
len_src_slices = len(src_slices)
new_src_grid_slice = [None] * len_src_slices
for idx in range(len_src_slices):
for rank_src_grid_slice in gathered_src_grid_slice:
if rank_src_grid_slice[idx] is not None:
new_src_grid_slice[idx] = rank_src_grid_slice[idx]
break
src_slices = new_src_grid_slice
with vm.scoped('index write', [0]):
if not vm.is_null:
dim = Dimension('nfiles', len(src_filenames))
vname = ['source_filename', 'destination_filename', 'weights_filename']
values = [src_filenames, dst_filenames, wgt_filenames]
grid_chunker_destination = GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE
attrs = [{'esmf_role': 'grid_chunker_source'},
{'esmf_role': grid_chunker_destination},
{'esmf_role': 'grid_chunker_weights'}]
vc = VariableCollection()
grid_chunker_index = GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE
vidx = Variable(name=grid_chunker_index)
vidx.attrs['esmf_role'] = grid_chunker_index
vidx.attrs['grid_chunker_source'] = 'source_filename'
vidx.attrs[GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE] = 'destination_filename'
vidx.attrs['grid_chunker_weights'] = 'weights_filename'
vidx.attrs[GridChunkerConstants.IndexFile.NAME_SRC_GRID_SHAPE] = src_global_shape
vidx.attrs[GridChunkerConstants.IndexFile.NAME_DST_GRID_SHAPE] = dst_global_shape
vc.add_variable(vidx)
for idx in range(len(vname)):
v = Variable(name=vname[idx], dimensions=dim, dtype=str, value=values[idx], attrs=attrs[idx])
vc.add_variable(v)
bounds_dimension = Dimension(name='bounds', size=2)
# TODO: This needs to work with four dimensions.
# Source -----------------------------------------------------------------------------------------------
self.src_grid._gc_create_index_bounds_(RegriddingRole.SOURCE, vidx, vc, src_slices, dim,
bounds_dimension)
# Destination ------------------------------------------------------------------------------------------
self.dst_grid._gc_create_index_bounds_(RegriddingRole.DESTINATION, vidx, vc, dst_slices, dim,
bounds_dimension)
vc.write(index_path)
vm.barrier()
def create_field(self, *args, **kwargs):
"""
Create a field object. In general, this should not be overloaded by subclasses.
:keyword bool format_time: ``(=True)`` If ``False``, do not convert numeric times to Python date objects.
:keyword str grid_abstraction: ``(='auto')`` If provided, use this grid abstraction.
:keyword raw_field: ``(=None)`` If provided, modify this field instead.
:type raw_field: None | :class:`~ocgis.Field`
:param kwargs: Additional keyword arguments to :meth:`~ocgis.driver.base.AbstractDriver.create_raw_field`.
:return: :class:`ocgis.Field`
"""
kwargs = kwargs.copy()
raw_field = kwargs.pop('raw_field', None)
format_time = kwargs.pop(KeywordArgument.FORMAT_TIME, True)
grid_abstraction = kwargs.pop(KeywordArgument.GRID_ABSTRACTION, self.rd.grid_abstraction)
grid_is_isomorphic = kwargs.pop('grid_is_isomorphic', self.rd.grid_is_isomorphic)
if raw_field is None:
# Get the raw variable collection from source.
new_kwargs = kwargs.copy()
new_kwargs['source_name'] = None
raw_field = self.create_raw_field(*args, **new_kwargs)
# Get the appropriate metadata for the collection.
group_metadata = self.get_group_metadata(raw_field.group, self.metadata_source)
# Always pull the dimension map from the request dataset. This allows it to be overloaded.
dimension_map = self.get_group_metadata(raw_field.group, self.rd.dimension_map)
# Modify the coordinate system variable. If it is overloaded on the request dataset, then the variable
# collection needs to be updated to hold the variable and any alternative coordinate systems needs to be
# removed.
to_remove = None
to_add = None
crs = self.get_crs(group_metadata)
if self.rd._has_assigned_coordinate_system:
to_add = self.rd._crs
if crs is not None:
to_remove = crs.name
else:
if self.rd._crs is not None and self.rd._crs != 'auto':
to_add = self.rd._crs
if crs is not None:
to_remove = crs.name
elif crs is not None:
to_add = crs
if to_remove is not None:
raw_field.pop(to_remove, None)
if to_add is not None:
raw_field.add_variable(to_add, force=True)
# Overload the dimension map with the CRS.
if to_add is not None:
# dimension_map[DimensionMapKey.CRS][DimensionMapKey.VARIABLE] = to_add.name
dimension_map.set_crs(to_add.name)
# Remove the mask variable if present in the raw dimension map and the source dimension map is set to None.
if self.rd.dimension_map.get_spatial_mask() is None and self.dimension_map_raw.get_spatial_mask() is not None:
raw_field.pop(self.dimension_map_raw.get_spatial_mask())
# Convert the raw variable collection to a field.
# TODO: Identify a way to remove this code block; field should be appropriately initialized; format_time and grid_abstraction are part of a dimension map.
kwargs[KeywordArgument.DIMENSION_MAP] = dimension_map
kwargs[KeywordArgument.FORMAT_TIME] = format_time
if grid_abstraction != 'auto':
kwargs[KeywordArgument.GRID_ABSTRACTION] = grid_abstraction
if grid_is_isomorphic != 'auto':
kwargs['grid_is_isomorphic'] = grid_is_isomorphic
field = Field.from_variable_collection(raw_field, *args, **kwargs)
# If this is a source grid for regridding, ensure the flag is updated.
field.regrid_source = self.rd.regrid_source
# Update the assigned coordinate system flag.
field._has_assigned_coordinate_system = self.rd._has_assigned_coordinate_system
# Apply any requested subsets.
if self.rd.time_range is not None:
field = field.time.get_between(*self.rd.time_range).parent
if self.rd.time_region is not None:
field = field.time.get_time_region(self.rd.time_region).parent
if self.rd.time_subset_func is not None:
field = field.time.get_subset_by_function(self.rd.time_subset_func).parent
if self.rd.level_range is not None:
field = field.level.get_between(*self.rd.level_range).parent
# These variables have all the dimensions needed for a data classification. Use overloaded values from the
# request dataset if they are provided.
try:
data_variable_names = list(get_variable_names(self.rd.rename_variable))
except NoDataVariablesFound:
# It is okay to have no data variables in a field.
data_variable_names = []
pass
for dvn in data_variable_names:
field.append_to_tags(TagName.DATA_VARIABLES, dvn, create=True)
# Load child fields.
for child in list(field.children.values()):
kwargs['raw_field'] = child
field.children[child.name] = self.create_field(*args, **kwargs)
return field
def test_write(self):
# Test writing a basic grid.
path = self.get_temporary_file_path('foo.nc')
x = Variable(name='x', value=[1, 2], dimensions='x')
y = Variable(name='y', value=[3, 4, 5, 6, 7], dimensions='y')
dmap = {'x': {'variable': 'x'}, 'y': {'variable': 'y'}}
field = Field(variables=[x, y], dimension_map=dmap)
desired_value_stacked = field.grid.get_value_stacked()
self.assertEqual(field.grid.parent['x'].get_value().shape, (2,))
self.assertTrue(field.grid.is_vectorized)
field.write(path)
out_field = RequestDataset(path).get()
self.assertTrue(out_field.grid.is_vectorized)
actual_value_stacked = out_field.grid.get_value_stacked()
self.assertNumpyAll(actual_value_stacked, desired_value_stacked)
# Test another grid.
grid = self.get_gridxy(crs=WGS84())
self.assertTrue(grid.is_vectorized)
field = Field(grid=grid)
self.assertTrue(field.grid.is_vectorized)
path = self.get_temporary_file_path('out.nc')
with self.nc_scope(path, 'w') as ds:
field.write(ds)
self.assertTrue(field.grid.is_vectorized)
with self.nc_scope(path) as ds:
if atleast_ncver("1.4"):
actual = grid.x.mv()
else:
actual = grid.x.v()
self.assertNumpyAll(ds.variables[grid.x.name][:], actual)
var = ds.variables[grid.y.name]
if atleast_ncver("1.4"):
actual = grid.y.mv()
else:
actual = grid.y.v()
self.assertNumpyAll(var[:], actual)
self.assertEqual(var.axis, 'Y')
self.assertIn(grid.crs.name, ds.variables)
# Test with 2-d x and y arrays.
grid = self.get_gridxy(with_2d_variables=True)
field = Field(grid=grid)
path = self.get_temporary_file_path('out.nc')
field.grid.set_extrapolated_bounds('xbounds', 'ybounds', 'bounds')
with self.nc_scope(path, 'w') as ds:
field.write(ds)
# self.ncdump(path)
with self.nc_scope(path) as ds:
var = ds.variables['y']
if atleast_ncver("1.4"):
actual = grid.y.mv()
else:
actual = grid.y.v()
self.assertNumpyAll(var[:], actual)
# Test writing a vectorized grid with corners.
grid = self.get_gridxy()
field = Field(grid=grid)
self.assertIsNotNone(field.grid.dimensions)
self.assertFalse(field.grid.has_bounds)
field.grid.set_extrapolated_bounds('xbnds', 'ybnds', 'corners')
self.assertTrue(field.grid.is_vectorized)
path = self.get_temporary_file_path('out.nc')
with self.nc_scope(path, 'w') as ds:
field.write(ds)
# self.ncdump(path)
with self.nc_scope(path, 'r') as ds:
self.assertEqual(['ydim'], [d for d in ds.variables['y'].dimensions])
self.assertEqual(['xdim'], [d for d in ds.variables['x'].dimensions])
