def get_meta(self):
meta_converter = MetaOCGISConverter(self)
rows = meta_converter.get_rows()
return '\n'.join(rows)
def write(self):
ocgis_lh('starting write method', self._log, logging.DEBUG)
# Indicates if user geometries should be written to file.
write_ugeom = False
# Path to the output object.
f = {KeywordArgument.PATH: self.path}
build = True
for i, coll in enumerate(self):
# This will be changed to "write" if we are on the build loop.
write_mode = MPIWriteMode.APPEND
if build:
# During a build loop, create the file and write the first series of records. Let the drivers determine
# the appropriate write modes for handling parallelism.
write_mode = None
# Write the user geometries if selected and there is one present on the incoming collection.
if self._add_ugeom and coll.has_container_geometries:
write_ugeom = True
if write_ugeom:
if vm.rank == 0:
# The output file name for the user geometries.
ugid_shp_name = self.prefix + '_ugid.shp'
if self._add_ugeom_nest:
ugeom_fiona_path = os.path.join(
self._get_or_create_shp_folder_(),
ugid_shp_name)
else:
ugeom_fiona_path = os.path.join(
self.outdir, ugid_shp_name)
else:
ugeom_fiona_path = None
build = False
f[KeywordArgument.WRITE_MODE] = write_mode
self._write_coll_(f, self._preformatting_(i, coll))
if write_ugeom:
with vm.scoped(SubcommName.UGEOM_WRITE, [0]):
if not vm.is_null:
for subset_field in list(coll.children.values()):
subset_field.write(ugeom_fiona_path,
write_mode=write_mode,
driver=DriverVector)
# The metadata and dataset descriptor files may only be written if OCGIS operations are present.
ops = self.ops
if ops is not None and self.add_auxiliary_files and MPI_RANK == 0:
# Add OCGIS metadata output if requested.
if self.add_meta:
ocgis_lh('adding OCGIS metadata file', 'conv', logging.DEBUG)
from ocgis.conv.meta import MetaOCGISConverter
lines = MetaOCGISConverter(ops).write()
out_path = os.path.join(
self.outdir,
self.prefix + '_' + MetaOCGISConverter._meta_filename)
with open(out_path, 'w') as f:
f.write(lines)
# Add the dataset descriptor file if requested.
if self._add_did_file:
ocgis_lh('writing dataset description (DID) file', 'conv',
logging.DEBUG)
path = os.path.join(self.outdir, self.prefix + '_did.csv')
_write_dataset_identifier_file_(path, ops)
# Add source metadata if requested.
if self._add_source_meta:
ocgis_lh('writing source metadata file', 'conv', logging.DEBUG)
path = os.path.join(self.outdir,
self.prefix + '_source_metadata.txt')
_write_source_meta_(path, ops)
# Return the internal path unless overloaded by subclasses.
ret = self._get_return_()
return ret
def test_write(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd)
conv = MetaOCGISConverter(ops)
self.assertTrue(len(conv.write()) > 4000)
self.assertEqual(len(os.listdir(self.current_dir_output)), 0)
def test_init(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd)
MetaOCGISConverter(ops)
def test_write(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd)
conv = MetaOCGISConverter(ops)
self.assertTrue(len(conv.write()) > 4000)
self.assertEqual(len(os.listdir(self.current_dir_output)), 0)
def write(self):
ocgis_lh('starting write method', self._log, logging.DEBUG)
# Indicates if user geometries should be written to file.
write_ugeom = False
ncoll = len(self.ops.geom)
build = True
for i, coll in enumerate(self):
ugids = coll.properties.keys()
assert len(ugids) == 1
ugid = ugids[0]
# Geometry centroid location
lon, lat = coll.geoms[ugid].centroid.xy
for field in coll.iter_fields():
lon_attrs = field.x.attrs.copy()
lat_attrs = field.y.attrs.copy()
# Removed for now. It'd be nice to find an elegant way to retain those.
field.remove_variable('lat')
field.remove_variable('lon')
# Create new lon and lat variables
field.add_variable(
ocgis.Variable('lon',
value=lon,
dimensions=(DimensionName.UNIONED_GEOMETRY,),
attrs=dict(lon_attrs, **{'long_name':'Centroid longitude'})
)
)
field.add_variable(
ocgis.Variable('lat',
value=lat,
dimensions=(DimensionName.UNIONED_GEOMETRY,),
attrs=dict(lat_attrs, **{'long_name':'Centroid latitude'})
)
)
if 'ocgis_spatial_mask' in field:
# Remove the spatial_mask and replace by new one.
field.remove_variable('ocgis_spatial_mask')
grid = ocgis.Grid(field['lon'], field['lat'], abstraction='point',
crs=field.crs, parent=field)
grid.set_mask([[False,]])
field.set_grid(grid)
# Geometry variables from the geom properties dict
# There is no metadata for those...
dm = get_data_model(self.ops)
for key, val in coll.properties[ugid].items():
if np.issubdtype(type(val), int):
dt = get_dtype('int', dm)
elif np.issubdtype(type(val), float):
dt = get_dtype('float', dm)
else:
dt='auto'
field.add_variable(
ocgis.Variable(key,
value=[val,],
dtype=dt,
dimensions=(DimensionName.UNIONED_GEOMETRY,)))
# ------------------ Dimension update ------------------------ #
# Modify the dimensions for the number of geometries
gdim = field.dimensions[DimensionName.UNIONED_GEOMETRY]
gdim.set_size(ncoll)
for var in field.iter_variables_by_dimensions([gdim]):
d = var.dimensions_dict[DimensionName.UNIONED_GEOMETRY]
d.bounds_local = (i, i+1)
# ------------------------------------------------------------ #
# CF-Conventions
# Can this be anything else than a timeseries_id
# Options are timeseries_id, profile_id, trajectory_id
gid = field[HeaderName.ID_GEOMETRY]
gid.attrs['cf_role'] = 'timeseries_id'
# TODO: Hard-code the name in constants.py
gdim.set_name('region')
# Path to the output object.
# I needed to put it here because _write_archetype pops it, so it's not available after the first loop.
f = {KeywordArgument.PATH: self.path}
# This will be changed to "write" if we are on the build loop.
write_mode = MPIWriteMode.APPEND
if build:
# During a build loop, create the file and write the first series of records. Let the drivers determine
# the appropriate write modes for handling parallelism.
write_mode = None
# Write the user geometries if selected and there is one present on the incoming collection.
if self._add_ugeom and coll.has_container_geometries:
write_ugeom = True
if write_ugeom:
if vm.rank == 0:
# The output file name for the user geometries.
ugid_shp_name = self.prefix + '_ugid.shp'
if self._add_ugeom_nest:
ugeom_fiona_path = os.path.join(self._get_or_create_shp_folder_(), ugid_shp_name)
else:
ugeom_fiona_path = os.path.join(self.outdir, ugid_shp_name)
else:
ugeom_fiona_path = None
build = False
f[KeywordArgument.WRITE_MODE] = write_mode
self._write_coll_(f, coll)
if write_ugeom:
with vm.scoped(SubcommName.UGEOM_WRITE, [0]):
if not vm.is_null:
for subset_field in list(coll.children.values()):
subset_field.write(ugeom_fiona_path, write_mode=write_mode, driver=DriverVector)
# The metadata and dataset descriptor files may only be written if OCGIS operations are present.
ops = self.ops
if ops is not None and self.add_auxiliary_files and MPI_RANK == 0:
# Add OCGIS metadata output if requested.
if self.add_meta:
ocgis_lh('adding OCGIS metadata file', 'conv', logging.DEBUG)
from ocgis.conv.meta import MetaOCGISConverter
lines = MetaOCGISConverter(ops).write()
out_path = os.path.join(self.outdir, self.prefix + '_' + MetaOCGISConverter._meta_filename)
with open(out_path, 'w') as f:
f.write(lines)
# Add the dataset descriptor file if requested.
if self._add_did_file:
ocgis_lh('writing dataset description (DID) file', 'conv', logging.DEBUG)
path = os.path.join(self.outdir, self.prefix + '_did.csv')
_write_dataset_identifier_file_(path, ops)
# Add source metadata if requested.
if self._add_source_meta:
ocgis_lh('writing source metadata file', 'conv', logging.DEBUG)
path = os.path.join(self.outdir, self.prefix + '_source_metadata.txt')
_write_source_meta_(path, ops)
# Return the internal path unless overloaded by subclasses.
ret = self._get_return_()
return ret
