def create_raw_field(self, group_metadata=None, group_name=None, name=None, source_name=constants.UNINITIALIZED, parent=None, uid=None): """ Create a raw field object. This field object should interpret metadata explicitly (i.e. no dimension map). In general this method should not be overloaded by subclasses. :param dict group_metadata: Metadata dictionary for the current group. :param str group_name: The name of the current group being processed. :param str name: See :class:`~ocgis.base.AbstractNamedObject` :param str source_name: See :class:`~ocgis.base.AbstractNamedObject` :param parent: :class:`~ocgis.variable.base.AbstractContainer` :param int uid: See :class:`~ocgis.variable.base.AbstractContainer` :return: :class:`~ocgis.Field` """ if group_metadata is None: group_metadata = self.rd.metadata field = Field(name=name, source_name=source_name, uid=uid, attrs=group_metadata.get('global_attributes')) for var in self.create_variables(group_metadata, parent=field).values(): field.add_variable(var, force=True) if parent is not None: parent.add_child(field) for k, v in group_metadata.get('groups', {}).items(): _ = self.create_raw_field(v, name=k, parent=field, group_name=k) return field
def get_ocgis_field_from_esmf_field(efield, field=None): """ :param efield: The ESMPy field object to convert to an OCGIS field. :type efield: :class:`ESMF.Field` :param field: If provided, use this as the template field for OCGIS field creation. :type field: :class:`~ocgis.Field` :return: :class:`~ocgis.Field` """ ometa = efield._ocgis dimnames = ometa.get('dimnames') dimnames_backref = ometa.get('dimnames_backref') ogrid = ometa.get('ocgis_grid') if ogrid is None: ogrid = get_ocgis_grid_from_esmf_grid(efield.grid) ovar = None if dimnames is not None and efield.name is not None: ovar = Variable(name=efield.name, value=efield.data, dimensions=dimnames, dtype=efield.data.dtype) broadcast_variable(ovar, get_dimension_names(dimnames_backref)) ovar.set_dimensions(dimnames_backref, force=True) if field is None: field = Field(grid=ogrid) else: field.set_grid(ogrid) if ovar is not None: field.add_variable(ovar, is_data=True, force=True) if ogrid.has_mask: field.grid.set_mask(ogrid.get_mask(), cascade=True) return field
def test_system_crs_and_grid_abstraction(self): f = Field(grid_abstraction='point') grid = self.get_gridxy(with_xy_bounds=True) f.add_variable(grid.x) crs = CoordinateReferenceSystem(epsg=2136, name='location') f.add_variable(crs) self.assertIsNone(f.crs) f.dimension_map.set_crs(crs) f.dimension_map.set_variable('x', grid.x) f.dimension_map.set_variable('y', grid.y) self.assertEqual(f.grid.crs, crs) f.set_geom(f.grid.get_abstraction_geometry()) self.assertEqual(f.grid.abstraction, 'point') self.assertEqual(f.geom.geom_type, 'Point')
def test_get_field_write_target(self): # Test coordinate system names are added to attributes of dimensioned variables. x = Variable('x', dimensions='x', value=[1]) y = Variable('y', dimensions='y', value=[2]) t = Variable('t', dimensions='t', value=[3]) crs = WGS84() d = Variable('data', dimensions=['t', 'y', 'x'], value=[[[1]]]) grid = Grid(x, y) field = Field(grid=grid, time=t, crs=crs) field.add_variable(d, is_data=True) target = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(target[d.name].attrs['grid_mapping'], crs.name) self.assertEqual(field.x.units, 'degrees_east') # Test bounds units are removed when writing. x = Variable(name='x', value=[1, 2, 3], dtype=float, dimensions='xdim', units='hours') y = Variable(name='y', value=[1, 2, 3], dtype=float, dimensions='ydim', units='hours') grid = Grid(x, y) grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds') self.assertEqual(x.bounds.units, x.units) self.assertEqual(y.bounds.units, y.units) field = Field(grid=grid) actual = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(x.bounds.units, x.units) self.assertNumpyMayShareMemory(actual[x.name].get_value(), field[x.name].get_value()) self.assertIsNone(actual[x.name].bounds.units) self.assertIsNone(actual[y.name].bounds.units) self.assertEqual(x.bounds.units, x.units) self.assertEqual(y.bounds.units, y.units) # Test actual coordinate system is triggered. field = Field() src = CFSpherical() dst = WGS84() field.set_crs(src) self.assertEqual(field.crs, src) self.assertIsNone(env.COORDSYS_ACTUAL) env.COORDSYS_ACTUAL = dst actual = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(actual.crs, dst) self.assertEqual(field.crs, src) self.assertNotIn(src.name, actual) self.assertIn(dst.name, actual)
def test_get_field_write_target(self): # Test coordinate system names are added to attributes of dimensioned variables. x = Variable('x', dimensions='x', value=[1]) y = Variable('y', dimensions='y', value=[2]) t = Variable('t', dimensions='t', value=[3]) crs = WGS84() d = Variable('data', dimensions=['t', 'y', 'x'], value=[[[1]]]) grid = Grid(x, y) field = Field(grid=grid, time=t, crs=crs) field.add_variable(d, is_data=True) target = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(target[d.name].attrs['grid_mapping'], crs.name) self.assertEqual(field.x.units, 'degrees_east') # Test bounds units are removed when writing. x = Variable(name='x', value=[1, 2, 3], dtype=float, dimensions='x', units='hours') y = Variable(name='y', value=[1, 2, 3], dtype=float, dimensions='x', units='hours') grid = Grid(x, y) grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds') self.assertEqual(x.bounds.units, x.units) self.assertEqual(y.bounds.units, y.units) field = Field(grid=grid) actual = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(x.bounds.units, x.units) self.assertNumpyMayShareMemory(actual[x.name].get_value(), field[x.name].get_value()) self.assertIsNone(actual[x.name].bounds.units) self.assertIsNone(actual[y.name].bounds.units) self.assertEqual(x.bounds.units, x.units) self.assertEqual(y.bounds.units, y.units)
def test_write_parallel(self): """Test writing by selective rank.""" if MPI_SIZE != 3 and MPI_SIZE != 1: raise SkipTest('MPI_SIZE != 1 or 3') ranks = list(range(MPI_SIZE)) for base_rank in ranks: for driver in [DriverCSV, DriverVector, DriverNetcdf]: if MPI_RANK == 0: path = self.get_temporary_file_path('{}-{}.{}'.format( driver.key, base_rank, driver.common_extension)) else: path = None path = MPI_COMM.bcast(path) with vm.scoped('field write by rank', [base_rank]): if not vm.is_null: geom = GeometryVariable( value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='geom') data = Variable(name='data', value=[10, 20], dimensions='geom') field = Field(geom=geom) field.add_variable(data, is_data=True) self.assertFalse(os.path.isdir(path)) field.write(path, driver=driver) self.assertFalse(os.path.isdir(path)) rd = RequestDataset(path, driver=driver) in_field = rd.get() self.assertEqual(in_field['data'].dimensions[0].size, 2) MPI_COMM.Barrier() MPI_COMM.Barrier()
def test_write_parallel(self): """Test writing by selective rank.""" if MPI_SIZE != 3 and MPI_SIZE != 1: raise SkipTest('MPI_SIZE != 1 or 3') ranks = list(range(MPI_SIZE)) for base_rank in ranks: for driver in [ DriverCSV, DriverVector, DriverNetcdf ]: if MPI_RANK == 0: path = self.get_temporary_file_path('{}-{}.{}'.format(driver.key, base_rank, driver.common_extension)) else: path = None path = MPI_COMM.bcast(path) with vm.scoped('field write by rank', [base_rank]): if not vm.is_null: geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='geom') data = Variable(name='data', value=[10, 20], dimensions='geom') field = Field(geom=geom) field.add_variable(data, is_data=True) self.assertFalse(os.path.isdir(path)) field.write(path, driver=driver) self.assertFalse(os.path.isdir(path)) rd = RequestDataset(path, driver=driver) in_field = rd.get() self.assertEqual(in_field['data'].dimensions[0].size, 2) MPI_COMM.Barrier() MPI_COMM.Barrier()
def _write_coll_(self, f, coll, add_geom_uid=True): ocgis_lh(msg='entering _write_coll_ in {}'.format(self.__class__), logger='csv-shp.converter', level=logging.DEBUG) # Load the geometries. The geometry identifier is needed for the data write. for field, container in coll.iter_fields(yield_container=True): field.set_abstraction_geom(create_ugid=True) # Write the output CSV file. ocgis_lh(msg='before CsvShapefileConverter super call in {}'.format( self.__class__), logger='csv-shp.converter', level=logging.DEBUG) super(CsvShapefileConverter, self)._write_coll_(f, coll, add_geom_uid=add_geom_uid) ocgis_lh(msg='after CsvShapefileConverter super call in {}'.format( self.__class__), logger='csv-shp.converter', level=logging.DEBUG) # The output geometry identifier shapefile path. if vm.rank == 0: fiona_path = os.path.join(self._get_or_create_shp_folder_(), self.prefix + '_gid.shp') else: fiona_path = None fiona_path = vm.bcast(fiona_path) if self.ops.aggregate: ocgis_lh( 'creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.', 'conv.csv-shp', logging.WARN) else: # Write the geometries for each container/field combination. for field, container in coll.iter_fields(yield_container=True): # The container may be empty. Only add the unique geometry identifier if the container has an # associated geometry. if container.geom is not None: ugid_var = Variable(name=container.geom.ugid.name, dimensions=field.geom.dimensions, dtype=constants.DEFAULT_NP_INT) ugid_var.get_value()[:] = container.geom.ugid.get_value( )[0] # Extract the variable components of the geometry file. geom = field.geom.copy() geom = geom.extract() if field.crs is not None: crs = field.crs.copy() crs = crs.extract() else: crs = None # If the dataset geometry identifier is not present, create it. gid = field[HeaderName.ID_GEOMETRY].copy() gid = gid.extract() # Construct the field to write. field_to_write = Field(geom=geom, crs=crs, uid=field.uid) if container.geom is not None: field_to_write.add_variable(ugid_var, is_data=True) field_to_write.add_variable(gid, is_data=True) # Maintain the field/dataset unique identifier if there is one. if field.uid is not None: if gid.repeat_record is None: rr = [] else: rr = list(gid.repeat_record) rr.append((HeaderName.DATASET_IDENTIFER, field.uid)) gid.repeat_record = rr # Write the field. field_to_write.write(fiona_path, write_mode=f[KeywordArgument.WRITE_MODE], driver=DriverKey.VECTOR)