def get_meta(self): meta_converter = MetaOCGISConverter(self) rows = meta_converter.get_rows() return '\n'.join(rows)
def write(self): ocgis_lh('starting write method', self._log, logging.DEBUG) # Indicates if user geometries should be written to file. write_ugeom = False # Path to the output object. f = {KeywordArgument.PATH: self.path} build = True for i, coll in enumerate(self): # This will be changed to "write" if we are on the build loop. write_mode = MPIWriteMode.APPEND if build: # During a build loop, create the file and write the first series of records. Let the drivers determine # the appropriate write modes for handling parallelism. write_mode = None # Write the user geometries if selected and there is one present on the incoming collection. if self._add_ugeom and coll.has_container_geometries: write_ugeom = True if write_ugeom: if vm.rank == 0: # The output file name for the user geometries. ugid_shp_name = self.prefix + '_ugid.shp' if self._add_ugeom_nest: ugeom_fiona_path = os.path.join( self._get_or_create_shp_folder_(), ugid_shp_name) else: ugeom_fiona_path = os.path.join( self.outdir, ugid_shp_name) else: ugeom_fiona_path = None build = False f[KeywordArgument.WRITE_MODE] = write_mode self._write_coll_(f, self._preformatting_(i, coll)) if write_ugeom: with vm.scoped(SubcommName.UGEOM_WRITE, [0]): if not vm.is_null: for subset_field in list(coll.children.values()): subset_field.write(ugeom_fiona_path, write_mode=write_mode, driver=DriverVector) # The metadata and dataset descriptor files may only be written if OCGIS operations are present. ops = self.ops if ops is not None and self.add_auxiliary_files and MPI_RANK == 0: # Add OCGIS metadata output if requested. if self.add_meta: ocgis_lh('adding OCGIS metadata file', 'conv', logging.DEBUG) from ocgis.conv.meta import MetaOCGISConverter lines = MetaOCGISConverter(ops).write() out_path = os.path.join( self.outdir, self.prefix + '_' + MetaOCGISConverter._meta_filename) with open(out_path, 'w') as f: f.write(lines) # Add the dataset descriptor file if requested. if self._add_did_file: ocgis_lh('writing dataset description (DID) file', 'conv', logging.DEBUG) path = os.path.join(self.outdir, self.prefix + '_did.csv') _write_dataset_identifier_file_(path, ops) # Add source metadata if requested. if self._add_source_meta: ocgis_lh('writing source metadata file', 'conv', logging.DEBUG) path = os.path.join(self.outdir, self.prefix + '_source_metadata.txt') _write_source_meta_(path, ops) # Return the internal path unless overloaded by subclasses. ret = self._get_return_() return ret
def test_write(self): rd = self.test_data.get_rd('cancm4_tas') ops = OcgOperations(dataset=rd) conv = MetaOCGISConverter(ops) self.assertTrue(len(conv.write()) > 4000) self.assertEqual(len(os.listdir(self.current_dir_output)), 0)
def test_init(self): rd = self.test_data.get_rd('cancm4_tas') ops = OcgOperations(dataset=rd) MetaOCGISConverter(ops)
def write(self): ocgis_lh('starting write method', self._log, logging.DEBUG) # Indicates if user geometries should be written to file. write_ugeom = False ncoll = len(self.ops.geom) build = True for i, coll in enumerate(self): ugids = coll.properties.keys() assert len(ugids) == 1 ugid = ugids[0] # Geometry centroid location lon, lat = coll.geoms[ugid].centroid.xy for field in coll.iter_fields(): lon_attrs = field.x.attrs.copy() lat_attrs = field.y.attrs.copy() # Removed for now. It'd be nice to find an elegant way to retain those. field.remove_variable('lat') field.remove_variable('lon') # Create new lon and lat variables field.add_variable( ocgis.Variable('lon', value=lon, dimensions=(DimensionName.UNIONED_GEOMETRY,), attrs=dict(lon_attrs, **{'long_name':'Centroid longitude'}) ) ) field.add_variable( ocgis.Variable('lat', value=lat, dimensions=(DimensionName.UNIONED_GEOMETRY,), attrs=dict(lat_attrs, **{'long_name':'Centroid latitude'}) ) ) if 'ocgis_spatial_mask' in field: # Remove the spatial_mask and replace by new one. field.remove_variable('ocgis_spatial_mask') grid = ocgis.Grid(field['lon'], field['lat'], abstraction='point', crs=field.crs, parent=field) grid.set_mask([[False,]]) field.set_grid(grid) # Geometry variables from the geom properties dict # There is no metadata for those... dm = get_data_model(self.ops) for key, val in coll.properties[ugid].items(): if np.issubdtype(type(val), int): dt = get_dtype('int', dm) elif np.issubdtype(type(val), float): dt = get_dtype('float', dm) else: dt='auto' field.add_variable( ocgis.Variable(key, value=[val,], dtype=dt, dimensions=(DimensionName.UNIONED_GEOMETRY,))) # ------------------ Dimension update ------------------------ # # Modify the dimensions for the number of geometries gdim = field.dimensions[DimensionName.UNIONED_GEOMETRY] gdim.set_size(ncoll) for var in field.iter_variables_by_dimensions([gdim]): d = var.dimensions_dict[DimensionName.UNIONED_GEOMETRY] d.bounds_local = (i, i+1) # ------------------------------------------------------------ # # CF-Conventions # Can this be anything else than a timeseries_id # Options are timeseries_id, profile_id, trajectory_id gid = field[HeaderName.ID_GEOMETRY] gid.attrs['cf_role'] = 'timeseries_id' # TODO: Hard-code the name in constants.py gdim.set_name('region') # Path to the output object. # I needed to put it here because _write_archetype pops it, so it's not available after the first loop. f = {KeywordArgument.PATH: self.path} # This will be changed to "write" if we are on the build loop. write_mode = MPIWriteMode.APPEND if build: # During a build loop, create the file and write the first series of records. Let the drivers determine # the appropriate write modes for handling parallelism. write_mode = None # Write the user geometries if selected and there is one present on the incoming collection. if self._add_ugeom and coll.has_container_geometries: write_ugeom = True if write_ugeom: if vm.rank == 0: # The output file name for the user geometries. ugid_shp_name = self.prefix + '_ugid.shp' if self._add_ugeom_nest: ugeom_fiona_path = os.path.join(self._get_or_create_shp_folder_(), ugid_shp_name) else: ugeom_fiona_path = os.path.join(self.outdir, ugid_shp_name) else: ugeom_fiona_path = None build = False f[KeywordArgument.WRITE_MODE] = write_mode self._write_coll_(f, coll) if write_ugeom: with vm.scoped(SubcommName.UGEOM_WRITE, [0]): if not vm.is_null: for subset_field in list(coll.children.values()): subset_field.write(ugeom_fiona_path, write_mode=write_mode, driver=DriverVector) # The metadata and dataset descriptor files may only be written if OCGIS operations are present. ops = self.ops if ops is not None and self.add_auxiliary_files and MPI_RANK == 0: # Add OCGIS metadata output if requested. if self.add_meta: ocgis_lh('adding OCGIS metadata file', 'conv', logging.DEBUG) from ocgis.conv.meta import MetaOCGISConverter lines = MetaOCGISConverter(ops).write() out_path = os.path.join(self.outdir, self.prefix + '_' + MetaOCGISConverter._meta_filename) with open(out_path, 'w') as f: f.write(lines) # Add the dataset descriptor file if requested. if self._add_did_file: ocgis_lh('writing dataset description (DID) file', 'conv', logging.DEBUG) path = os.path.join(self.outdir, self.prefix + '_did.csv') _write_dataset_identifier_file_(path, ops) # Add source metadata if requested. if self._add_source_meta: ocgis_lh('writing source metadata file', 'conv', logging.DEBUG) path = os.path.join(self.outdir, self.prefix + '_source_metadata.txt') _write_source_meta_(path, ops) # Return the internal path unless overloaded by subclasses. ret = self._get_return_() return ret