def create_raw_field(self,
group_metadata=None,
group_name=None,
name=None,
source_name=constants.UNINITIALIZED,
parent=None,
uid=None):
"""
Create a raw field object. This field object should interpret metadata explicitly (i.e. no dimension map). In
general this method should not be overloaded by subclasses.
:param dict group_metadata: Metadata dictionary for the current group.
:param str group_name: The name of the current group being processed.
:param str name: See :class:`~ocgis.base.AbstractNamedObject`
:param str source_name: See :class:`~ocgis.base.AbstractNamedObject`
:param parent: :class:`~ocgis.variable.base.AbstractContainer`
:param int uid: See :class:`~ocgis.variable.base.AbstractContainer`
:return: :class:`~ocgis.Field`
"""
if group_metadata is None:
group_metadata = self.rd.metadata
field = Field(name=name,
source_name=source_name,
uid=uid,
attrs=group_metadata.get('global_attributes'))
for var in self.create_variables(group_metadata,
parent=field).values():
field.add_variable(var, force=True)
if parent is not None:
parent.add_child(field)
for k, v in group_metadata.get('groups', {}).items():
_ = self.create_raw_field(v, name=k, parent=field, group_name=k)
return field
def create_raw_field(self, group_metadata=None, group_name=None, name=None, source_name=constants.UNINITIALIZED,
parent=None, uid=None):
"""
Create a raw field object. This field object should interpret metadata explicitly (i.e. no dimension map). In
general this method should not be overloaded by subclasses.
:param dict group_metadata: Metadata dictionary for the current group.
:param str group_name: The name of the current group being processed.
:param str name: See :class:`~ocgis.base.AbstractNamedObject`
:param str source_name: See :class:`~ocgis.base.AbstractNamedObject`
:param parent: :class:`~ocgis.variable.base.AbstractContainer`
:param int uid: See :class:`~ocgis.variable.base.AbstractContainer`
:return: :class:`~ocgis.Field`
"""
if group_metadata is None:
group_metadata = self.rd.metadata
field = Field(name=name, source_name=source_name, uid=uid, attrs=group_metadata.get('global_attributes'))
for var in self.create_variables(group_metadata, parent=field).values():
field.add_variable(var, force=True)
if parent is not None:
parent.add_child(field)
for k, v in group_metadata.get('groups', {}).items():
_ = self.create_raw_field(v, name=k, parent=field, group_name=k)
return field
def get_ocgis_field_from_esmf_field(efield, field=None):
"""
:param efield: The ESMPy field object to convert to an OCGIS field.
:type efield: :class:`ESMF.Field`
:param field: If provided, use this as the template field for OCGIS field creation.
:type field: :class:`~ocgis.Field`
:return: :class:`~ocgis.Field`
"""
ometa = efield._ocgis
dimnames = ometa.get('dimnames')
dimnames_backref = ometa.get('dimnames_backref')
ogrid = ometa.get('ocgis_grid')
if ogrid is None:
ogrid = get_ocgis_grid_from_esmf_grid(efield.grid)
ovar = None
if dimnames is not None and efield.name is not None:
ovar = Variable(name=efield.name, value=efield.data, dimensions=dimnames, dtype=efield.data.dtype)
broadcast_variable(ovar, get_dimension_names(dimnames_backref))
ovar.set_dimensions(dimnames_backref, force=True)
if field is None:
field = Field(grid=ogrid)
else:
field.set_grid(ogrid)
if ovar is not None:
field.add_variable(ovar, is_data=True, force=True)
if ogrid.has_mask:
field.grid.set_mask(ogrid.get_mask(), cascade=True)
return field
def test_system_crs_and_grid_abstraction(self):
f = Field(grid_abstraction='point')
grid = self.get_gridxy(with_xy_bounds=True)
f.add_variable(grid.x)
crs = CoordinateReferenceSystem(epsg=2136, name='location')
f.add_variable(crs)
self.assertIsNone(f.crs)
f.dimension_map.set_crs(crs)
f.dimension_map.set_variable('x', grid.x)
f.dimension_map.set_variable('y', grid.y)
self.assertEqual(f.grid.crs, crs)
f.set_geom(f.grid.get_abstraction_geometry())
self.assertEqual(f.grid.abstraction, 'point')
self.assertEqual(f.geom.geom_type, 'Point')
def test_system_crs_and_grid_abstraction(self):
f = Field(grid_abstraction='point')
grid = self.get_gridxy(with_xy_bounds=True)
f.add_variable(grid.x)
crs = CoordinateReferenceSystem(epsg=2136, name='location')
f.add_variable(crs)
self.assertIsNone(f.crs)
f.dimension_map.set_crs(crs)
f.dimension_map.set_variable('x', grid.x)
f.dimension_map.set_variable('y', grid.y)
self.assertEqual(f.grid.crs, crs)
f.set_geom(f.grid.get_abstraction_geometry())
self.assertEqual(f.grid.abstraction, 'point')
self.assertEqual(f.geom.geom_type, 'Point')
def test_get_field_write_target(self):
# Test coordinate system names are added to attributes of dimensioned variables.
x = Variable('x', dimensions='x', value=[1])
y = Variable('y', dimensions='y', value=[2])
t = Variable('t', dimensions='t', value=[3])
crs = WGS84()
d = Variable('data', dimensions=['t', 'y', 'x'], value=[[[1]]])
grid = Grid(x, y)
field = Field(grid=grid, time=t, crs=crs)
field.add_variable(d, is_data=True)
target = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(target[d.name].attrs['grid_mapping'], crs.name)
self.assertEqual(field.x.units, 'degrees_east')
# Test bounds units are removed when writing.
x = Variable(name='x', value=[1, 2, 3], dtype=float, dimensions='xdim', units='hours')
y = Variable(name='y', value=[1, 2, 3], dtype=float, dimensions='ydim', units='hours')
grid = Grid(x, y)
grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds')
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
field = Field(grid=grid)
actual = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(x.bounds.units, x.units)
self.assertNumpyMayShareMemory(actual[x.name].get_value(), field[x.name].get_value())
self.assertIsNone(actual[x.name].bounds.units)
self.assertIsNone(actual[y.name].bounds.units)
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
# Test actual coordinate system is triggered.
field = Field()
src = CFSpherical()
dst = WGS84()
field.set_crs(src)
self.assertEqual(field.crs, src)
self.assertIsNone(env.COORDSYS_ACTUAL)
env.COORDSYS_ACTUAL = dst
actual = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(actual.crs, dst)
self.assertEqual(field.crs, src)
self.assertNotIn(src.name, actual)
self.assertIn(dst.name, actual)
def test_get_field_write_target(self):
# Test coordinate system names are added to attributes of dimensioned variables.
x = Variable('x', dimensions='x', value=[1])
y = Variable('y', dimensions='y', value=[2])
t = Variable('t', dimensions='t', value=[3])
crs = WGS84()
d = Variable('data', dimensions=['t', 'y', 'x'], value=[[[1]]])
grid = Grid(x, y)
field = Field(grid=grid, time=t, crs=crs)
field.add_variable(d, is_data=True)
target = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(target[d.name].attrs['grid_mapping'], crs.name)
self.assertEqual(field.x.units, 'degrees_east')
# Test bounds units are removed when writing.
x = Variable(name='x',
value=[1, 2, 3],
dtype=float,
dimensions='x',
units='hours')
y = Variable(name='y',
value=[1, 2, 3],
dtype=float,
dimensions='x',
units='hours')
grid = Grid(x, y)
grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds')
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
field = Field(grid=grid)
actual = DriverNetcdfCF._get_field_write_target_(field)
self.assertEqual(x.bounds.units, x.units)
self.assertNumpyMayShareMemory(actual[x.name].get_value(),
field[x.name].get_value())
self.assertIsNone(actual[x.name].bounds.units)
self.assertIsNone(actual[y.name].bounds.units)
self.assertEqual(x.bounds.units, x.units)
self.assertEqual(y.bounds.units, y.units)
def test_write_parallel(self):
"""Test writing by selective rank."""
if MPI_SIZE != 3 and MPI_SIZE != 1:
raise SkipTest('MPI_SIZE != 1 or 3')
ranks = list(range(MPI_SIZE))
for base_rank in ranks:
for driver in [DriverCSV, DriverVector, DriverNetcdf]:
if MPI_RANK == 0:
path = self.get_temporary_file_path('{}-{}.{}'.format(
driver.key, base_rank, driver.common_extension))
else:
path = None
path = MPI_COMM.bcast(path)
with vm.scoped('field write by rank', [base_rank]):
if not vm.is_null:
geom = GeometryVariable(
value=[Point(1, 2), Point(3, 4)],
name='geom',
dimensions='geom')
data = Variable(name='data',
value=[10, 20],
dimensions='geom')
field = Field(geom=geom)
field.add_variable(data, is_data=True)
self.assertFalse(os.path.isdir(path))
field.write(path, driver=driver)
self.assertFalse(os.path.isdir(path))
rd = RequestDataset(path, driver=driver)
in_field = rd.get()
self.assertEqual(in_field['data'].dimensions[0].size,
2)
MPI_COMM.Barrier()
MPI_COMM.Barrier()
def get_ocgis_field_from_esmf_field(efield, field=None):
"""
:param efield: The ESMPy field object to convert to an OCGIS field.
:type efield: :class:`ESMF.Field`
:param field: If provided, use this as the template field for OCGIS field creation.
:type field: :class:`~ocgis.Field`
:return: :class:`~ocgis.Field`
"""
ometa = efield._ocgis
dimnames = ometa.get('dimnames')
dimnames_backref = ometa.get('dimnames_backref')
ogrid = ometa.get('ocgis_grid')
if ogrid is None:
ogrid = get_ocgis_grid_from_esmf_grid(efield.grid)
ovar = None
if dimnames is not None and efield.name is not None:
ovar = Variable(name=efield.name,
value=efield.data,
dimensions=dimnames,
dtype=efield.data.dtype)
broadcast_variable(ovar, get_dimension_names(dimnames_backref))
ovar.set_dimensions(dimnames_backref, force=True)
if field is None:
field = Field(grid=ogrid)
else:
field.set_grid(ogrid)
if ovar is not None:
field.add_variable(ovar, is_data=True, force=True)
if ogrid.has_mask:
field.grid.set_mask(ogrid.get_mask(), cascade=True)
return field
def test_write_parallel(self):
"""Test writing by selective rank."""
if MPI_SIZE != 3 and MPI_SIZE != 1:
raise SkipTest('MPI_SIZE != 1 or 3')
ranks = list(range(MPI_SIZE))
for base_rank in ranks:
for driver in [
DriverCSV,
DriverVector,
DriverNetcdf
]:
if MPI_RANK == 0:
path = self.get_temporary_file_path('{}-{}.{}'.format(driver.key, base_rank,
driver.common_extension))
else:
path = None
path = MPI_COMM.bcast(path)
with vm.scoped('field write by rank', [base_rank]):
if not vm.is_null:
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='geom')
data = Variable(name='data', value=[10, 20], dimensions='geom')
field = Field(geom=geom)
field.add_variable(data, is_data=True)
self.assertFalse(os.path.isdir(path))
field.write(path, driver=driver)
self.assertFalse(os.path.isdir(path))
rd = RequestDataset(path, driver=driver)
in_field = rd.get()
self.assertEqual(in_field['data'].dimensions[0].size, 2)
MPI_COMM.Barrier()
MPI_COMM.Barrier()
def _write_coll_(self, f, coll, add_geom_uid=True):
ocgis_lh(msg='entering _write_coll_ in {}'.format(self.__class__),
logger='csv-shp.converter',
level=logging.DEBUG)
# Load the geometries. The geometry identifier is needed for the data write.
for field, container in coll.iter_fields(yield_container=True):
field.set_abstraction_geom(create_ugid=True)
# Write the output CSV file.
ocgis_lh(msg='before CsvShapefileConverter super call in {}'.format(
self.__class__),
logger='csv-shp.converter',
level=logging.DEBUG)
super(CsvShapefileConverter,
self)._write_coll_(f, coll, add_geom_uid=add_geom_uid)
ocgis_lh(msg='after CsvShapefileConverter super call in {}'.format(
self.__class__),
logger='csv-shp.converter',
level=logging.DEBUG)
# The output geometry identifier shapefile path.
if vm.rank == 0:
fiona_path = os.path.join(self._get_or_create_shp_folder_(),
self.prefix + '_gid.shp')
else:
fiona_path = None
fiona_path = vm.bcast(fiona_path)
if self.ops.aggregate:
ocgis_lh(
'creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.',
'conv.csv-shp', logging.WARN)
else:
# Write the geometries for each container/field combination.
for field, container in coll.iter_fields(yield_container=True):
# The container may be empty. Only add the unique geometry identifier if the container has an
# associated geometry.
if container.geom is not None:
ugid_var = Variable(name=container.geom.ugid.name,
dimensions=field.geom.dimensions,
dtype=constants.DEFAULT_NP_INT)
ugid_var.get_value()[:] = container.geom.ugid.get_value(
)[0]
# Extract the variable components of the geometry file.
geom = field.geom.copy()
geom = geom.extract()
if field.crs is not None:
crs = field.crs.copy()
crs = crs.extract()
else:
crs = None
# If the dataset geometry identifier is not present, create it.
gid = field[HeaderName.ID_GEOMETRY].copy()
gid = gid.extract()
# Construct the field to write.
field_to_write = Field(geom=geom, crs=crs, uid=field.uid)
if container.geom is not None:
field_to_write.add_variable(ugid_var, is_data=True)
field_to_write.add_variable(gid, is_data=True)
# Maintain the field/dataset unique identifier if there is one.
if field.uid is not None:
if gid.repeat_record is None:
rr = []
else:
rr = list(gid.repeat_record)
rr.append((HeaderName.DATASET_IDENTIFER, field.uid))
gid.repeat_record = rr
# Write the field.
field_to_write.write(fiona_path,
write_mode=f[KeywordArgument.WRITE_MODE],
driver=DriverKey.VECTOR)
