def test_system_spatial_averaging_through_operations(self): data_name = 'data' with vm.scoped('write', [0]): if not vm.is_null: x = Variable('x', range(5), 'x', float) y = Variable('y', range(7), 'y', float) grid = Grid(x, y) data_value = np.arange(x.size * y.size).reshape(grid.shape) data = Variable(data_name, data_value, grid.dimensions, float) data_value = data.get_value() field = Field(grid=grid, is_data=data) path = self.get_temporary_file_path('data.nc') field.write(path) else: data_value, path = None, None data_value = MPI_COMM.bcast(data_value) path = MPI_COMM.bcast(path) rd = RequestDataset(path, variable=data_name) ops = OcgOperations(dataset=rd, aggregate=True) ret = ops.execute() if ret is None: self.assertNotEqual(vm.rank, vm.root) else: out_field = ret.get_element() if MPI_RANK == 0: desired = data_value.mean() actual = out_field.data_variables[0].get_value()[0] self.assertEqual(actual, desired)
def get_ocgis_field_from_esmf_field(efield, field=None): """ :param efield: The ESMPy field object to convert to an OCGIS field. :type efield: :class:`ESMF.Field` :param field: If provided, use this as the template field for OCGIS field creation. :type field: :class:`~ocgis.Field` :return: :class:`~ocgis.Field` """ ometa = efield._ocgis dimnames = ometa.get('dimnames') dimnames_backref = ometa.get('dimnames_backref') ogrid = ometa.get('ocgis_grid') if ogrid is None: ogrid = get_ocgis_grid_from_esmf_grid(efield.grid) ovar = None if dimnames is not None and efield.name is not None: ovar = Variable(name=efield.name, value=efield.data, dimensions=dimnames, dtype=efield.data.dtype) broadcast_variable(ovar, get_dimension_names(dimnames_backref)) ovar.set_dimensions(dimnames_backref, force=True) if field is None: field = Field(grid=ogrid) else: field.set_grid(ogrid) if ovar is not None: field.add_variable(ovar, is_data=True, force=True) if ogrid.has_mask: field.grid.set_mask(ogrid.get_mask(), cascade=True) return field
def test_system_dataset_identifiers_on_variables(self): """Test dataset identifiers make it to output variables for iteration.""" paths = [] variables = [] for suffix in [1, 2]: path = self.get_temporary_file_path('foo{}.nc'.format(suffix)) paths.append(path) x = Variable(name='x{}'.format(suffix), value=[2, 3], dimensions='x') y = Variable(name='y{}'.format(suffix), value=[4, 5, 6], dimensions='y') data_variable_name = 'data{}'.format(suffix) variables.append(data_variable_name) data = Variable(name=data_variable_name, value=np.arange(6).reshape(2, 3) + suffix, dimensions=['x', 'y']) grid = Grid(x, y) field = Field(grid=grid, is_data=data) field.write(path) rds = [RequestDataset(uri=p, variable=dv) for p, dv in zip(paths, variables)] ops = OcgOperations(dataset=rds) rds_uids = [ds.uid for ds in ops.dataset] self.assertEqual(rds_uids, [1, 2]) ret = ops.execute() for field in ret.iter_fields(): self.assertFalse(field.grid.has_allocated_abstraction_geometry) for variable in list(field.values()): if isinstance(variable, CoordinateReferenceSystem): continue self.assertIsNotNone(variable._request_dataset.uid) for row in variable.get_iter(): self.assertIsNotNone(row[HeaderName.DATASET_IDENTIFER])
def test_system_get_field_dimensioned_variables(self): """Test data is appropriately tagged to identify dimensioned variables.""" path = self.get_temporary_file_path('foo.nc') time = TemporalVariable(value=[1, 2, 3], dimensions='time') x = Variable(name='x', value=[10, 20], dimensions='x') y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y') data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time']) field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3]) field.write(path) # Test dimensioned variables are read from a file with appropriate metadata. rd = RequestDataset(path) self.assertEqual(rd.variable, ('data1', 'data2')) read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data1', 'data2')) # Test dimensioned variables are overloaded. rd = RequestDataset(path, variable='data2') read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data2', ))
def create_raw_field(self, group_metadata=None, group_name=None, name=None, source_name=constants.UNINITIALIZED, parent=None, uid=None): """ Create a raw field object. This field object should interpret metadata explicitly (i.e. no dimension map). In general this method should not be overloaded by subclasses. :param dict group_metadata: Metadata dictionary for the current group. :param str group_name: The name of the current group being processed. :param str name: See :class:`~ocgis.base.AbstractNamedObject` :param str source_name: See :class:`~ocgis.base.AbstractNamedObject` :param parent: :class:`~ocgis.variable.base.AbstractContainer` :param int uid: See :class:`~ocgis.variable.base.AbstractContainer` :return: :class:`~ocgis.Field` """ if group_metadata is None: group_metadata = self.rd.metadata field = Field(name=name, source_name=source_name, uid=uid, attrs=group_metadata.get('global_attributes')) for var in self.create_variables(group_metadata, parent=field).values(): field.add_variable(var, force=True) if parent is not None: parent.add_child(field) for k, v in group_metadata.get('groups', {}).items(): _ = self.create_raw_field(v, name=k, parent=field, group_name=k) return field
def test_system_create_field_dimensioned_variables(self): """Test data is appropriately tagged to identify dimensioned variables.""" path = self.get_temporary_file_path('foo.nc') time = TemporalVariable(value=[1, 2, 3], dimensions='time') x = Variable(name='x', value=[10, 20], dimensions='x') y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y') data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time']) field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3]) field.write(path) # Test dimensioned variables are read from a file with appropriate metadata. rd = RequestDataset(path) self.assertEqual(rd.variable, ('data1', 'data2')) read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data1', 'data2')) # Test dimensioned variables are overloaded. rd = RequestDataset(path, variable='data2') read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data2',))
def test_grid(self): # Test mask variable information is propagated through property. grid = self.get_gridxy(with_xy_bounds=True) self.assertTrue(grid.is_vectorized) self.assertTrue(grid.has_bounds) np.random.seed(1) value = np.random.rand(*grid.shape) select = value > 0.4 mask_var = create_spatial_mask_variable('nonstandard', select, grid.dimensions) grid.set_mask(mask_var) field = Field(grid=grid) self.assertTrue(field.grid.has_bounds) self.assertEqual(field.dimension_map.get_spatial_mask(), mask_var.name) self.assertNumpyAll(field.grid.get_mask(), mask_var.get_mask()) # Test dimension map bounds are updated appropriately. dim = Dimension('count', 2) x = Variable(name='x', value=[1., 2.], dimensions=dim) y = Variable(name='y', value=[1., 2.], dimensions=dim) xb = Variable(name='xb', value=[[0., 1.5], [1.5, 2.5]], dimensions=[dim, 'bounds']) yb = Variable(name='yb', value=[[0., 1.5], [1.5, 2.5]], dimensions=[dim, 'bounds']) variables = [x, y, xb, yb] dmap = DimensionMap() dmap.set_variable(DMK.X, x, bounds=xb) dmap.set_variable(DMK.Y, y, bounds=yb) f = Field(dimension_map=dmap, variables=variables) self.assertTrue(f.grid.has_bounds)
def test_set_geom(self): f = Field() self.assertIsNone(f.crs) g = GeometryVariable(value=[Point(1, 2)], dimensions='geom', crs=Spherical()) f.set_geom(g)
def test_init(self): field = self.get_ocgfield() self.assertIsInstance(field, Field) # Test unique identifier. field = Field(uid=4) self.assertEqual(field.uid, 4) # Test with a coordinate system and geometry. desired_crs = WGS84() geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='geom') field = Field(crs=desired_crs, geom=geom) self.assertEqual(field.crs, desired_crs) # Test dimension names are automatically added to dimension map. g = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='the_geom_dim') f = Field(geom=g) actual = f.dimension_map.get_dimension(DimensionMapKey.GEOM) self.assertEqual(actual, ['the_geom_dim']) # Test dimension map does not have any entries at initialization. actual = Field() desired = actual.dimension_map.as_dict() self.assertEqual(len(desired), 0)
def test_system_with_time_data(self): """Test writing data with a time dimension.""" path = self.get_temporary_file_path('what.shp') t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time') geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='time') field = Field(variables=[t, geom], dimension_map={ 'time': { 'variable': 'time' }, 'geom': { 'variable': 'geom' } }) field.write(path, iter_kwargs={'variable': 'time'}, driver=DriverVector) rd = RequestDataset(uri=path) field2 = rd.get() # netcdftime worthlessness poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'], ['1-01-02 12:00:00', '1-01-03 12:00:00']] actual = field2['TIME'].get_value().tolist() res = [p == actual for p in poss] self.assertTrue(any(res))
def test_get_by_tag(self): v1 = Variable(name='tas') v2 = Variable(name='tasmax') v3 = Variable(name='tasmin') tags = {'avg': ['tas'], 'other': ['tasmax', 'tasmin']} field = Field(variables=[v1, v2, v3], tags=tags) t = field.get_by_tag('other') self.assertAsSetEqual([ii.name for ii in t], tags['other'])
def test_write_variable_collection_different_data_types(self): """Test multiple data types are handled by the shapefile write when melted is True.""" v_int = Variable(name='an_int', value=[1, 2, 3], dtype=int, dimensions='three') v_flt = Variable(name='a_float', value=[10., 20., 30.], dtype=float, dimensions='three') g = GeometryVariable(name='points', value=[Point(1, 2), Point(3, 4), Point(5, 6)], dimensions='three') field = Field(is_data=[v_int, v_flt], geom=g) self.assertEqual(len(field.data_variables), 2) path = self.get_temporary_file_path('foo.shp') field.write(path, driver='vector', iter_kwargs=dict(melted=True))
def test_iter_masking_and_driver(self): """Test mask is set to None.""" time = TemporalVariable(value=[3, 4, 5], dimensions='time') data = Variable(value=[7, 8, 9], name='data', mask=[False, True, False], dimensions='time') field = Field(time=time, is_data=data, variables=data) itr = field.iter(allow_masked=True) actual = list(itr) self.assertIsNone(actual[1][1][data.name])
def test_system_conform_units(self): """Test conforming units on data read from shapefile.""" path = self.get_temporary_file_path('temps.shp') gvar = GeometryVariable(value=[Point(1, 2), Point(3, 4)], dimensions='g', name='geom') var = Variable(name='temp', value=[10., 20.], dimensions='g') field = Field(variables=[gvar, var], geom=gvar, is_data=var) field.write(path, driver=DriverVector) field = RequestDataset(path, units='celsius', variable='temp', conform_units_to='fahrenheit').get() self.assertNumpyAllClose(field['temp'].get_value(), np.array([50., 68.]))
def test_update_crs(self): # Test copying allows the CRS to be updated on the copy w/out changing the source CRS. desired = Spherical() gvar = GeometryVariable(value=[Point(1, 2)], name='geom', dimensions='geom') field = Field(crs=desired, geom=gvar) cfield = field.copy() self.assertEqual(cfield.crs, desired) new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326) cfield.update_crs(new_crs) self.assertEqual(field.crs, desired)
def test_write_variable_collection(self): # Attempt to write without a geometry variable. v = Variable('a', value=[1, 2], dimensions='bb') field = Field(variables=v) path = self.get_temporary_file_path('out.shp') with self.assertRaises(ValueError): field.write(path, driver=DriverVector) # Test writing a field with two-dimensional geometry storage. value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)] gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms') gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)]) var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a']) var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat']) var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon']) var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat']) field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data']) path = self.get_temporary_file_path('2d.shp') field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector) read = RequestDataset(uri=path).get() self.assertTrue(len(read) > 2) self.assertEqual(list(read.keys()), ['data', 'some_lats', 'some_lons', constants.VariableName.GEOMETRY_VARIABLE]) # Test writing a subset of the variables. path = self.get_temporary_file_path('limited.shp') value = [Point(1, 2), Point(3, 4), Point(5, 6)] gvar = GeometryVariable(value=value, name='points', dimensions='points') var1 = Variable('keep', value=[1, 2, 3], dimensions='points') var2 = Variable('remove', value=[4, 5, 6], dimensions='points') field = Field(variables=[var1, var2], geom=gvar, is_data=[var1]) field.write(path, variable_names=['keep'], driver=DriverVector) read = RequestDataset(uri=path).get() self.assertNotIn('remove', read) # Test using append. path = self.get_temporary_file_path('limited.shp') value = [Point(1, 2), Point(3, 4), Point(5, 6)] gvar = GeometryVariable(value=value, name='points', dimensions='points') var1 = Variable('keep', value=[1, 2, 3], dimensions='points') var2 = Variable('remove', value=[4, 5, 6], dimensions='points') field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2]) for idx in range(3): sub = field[{'points': idx}] if idx == 0: write_mode = MPIWriteMode.WRITE else: write_mode = MPIWriteMode.APPEND sub.write(path, write_mode=write_mode, driver=DriverVector) self.assertOGRFileLength(path, idx + 1)
def parse(self, value): if type(value) in [list, tuple]: if all([isinstance(element, dict) for element in value]): for ii, element in enumerate(value, start=1): if 'geom' not in element: ocgis_lh(exc=DefinitionValidationError(self, 'Geometry dictionaries must have a "geom" key.')) if 'properties' not in element: element['properties'] = {self._ugid_key: ii} crs = element.get('crs', constants.UNINITIALIZED) if 'crs' not in element: ocgis_lh(msg='No CRS in geometry dictionary - assuming WGS84.', level=logging.WARN) ret = Field.from_records(value, crs=crs, uid=self.geom_uid, union=self.union, data_model=self.data_model) else: if len(value) == 2: geom = Point(value[0], value[1]) elif len(value) == 4: minx, miny, maxx, maxy = value geom = Polygon(((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny))) if not geom.is_valid: raise DefinitionValidationError(self, 'Parsed geometry is not valid.') ret = [{'geom': geom, 'properties': {self._ugid_key: 1}}] ret = Field.from_records(ret, uid=self.geom_uid, union=self.union, data_model=self.data_model) self._bounds = geom.bounds elif isinstance(value, GeomCabinetIterator): self._shp_key = value.key or value.path # Always yield fields. value.as_field = True ret = value elif isinstance(value, BaseGeometry): ret = [{'geom': value, 'properties': {self._ugid_key: 1}}] ret = Field.from_records(ret, uid=self.geom_uid, union=self.union, data_model=self.data_model) elif value is None: ret = value elif isinstance(value, Field): ret = value elif isinstance(value, GeometryVariable): if value.ugid is None: msg = 'Geometry variables must have an associated "UGID".' raise DefinitionValidationError(self, msg) ret = Field(geom=value, crs=value.crs) else: raise NotImplementedError(type(value)) # Convert to singular field if this is a field object. if isinstance(ret, Field): ret = tuple(self._iter_singular_fields_(ret)) return ret
def test_system_parallel_write_ndvariable(self): """Test a parallel vector GIS write with a n-dimensional variable.""" ompi = OcgDist() ompi.create_dimension('time', 3) ompi.create_dimension('extra', 2) ompi.create_dimension('x', 4) ompi.create_dimension('y', 7, dist=True) ompi.update_dimension_bounds() if MPI_RANK == 0: path = self.get_temporary_file_path('foo.shp') t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time') t.set_extrapolated_bounds('the_time_bounds', 'bounds') extra = Variable(name='extra', value=[7, 8], dimensions='extra') x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float) x.set_extrapolated_bounds('x_bounds', 'bounds') # This will have the distributed dimension. y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float) y.set_extrapolated_bounds('y_bounds', 'bounds') data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x']) dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'}, 'y': {'variable': 'y', 'bounds': 'y_bounds'}, 'time': {'variable': 'time', 'bounds': 'the_time_bounds'}} vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data') vc.set_abstraction_geom() else: path, vc = [None] * 2 path = MPI_COMM.bcast(path) vc = variable_collection_scatter(vc, ompi) with vm.scoped_by_emptyable('write', vc): if not vm.is_null: vc.write(path, driver=DriverVector) MPI_COMM.Barrier() desired = 168 rd = RequestDataset(path, driver=DriverVector) sizes = MPI_COMM.gather(rd.get().geom.shape[0]) if MPI_RANK == 0: self.assertEqual(sum(sizes), desired)
def test_get_dimension_map_2d_spatial_coordinates(self): grid = create_gridxy_global() grid.expand() path = self.get_temporary_file_path('foo.nc') f = Field(grid=grid) f.write(path) rd = RequestDataset(path) field = rd.get() sub = field.get_field_slice({'y': 10, 'x': 5}) self.assertEqual(sub.grid.x.shape, (1, 1)) actual = f.dimension_map.get_dimension(DimensionMapKey.Y) self.assertEqual(actual, ['y']) actual = f.dimension_map.get_dimension(DimensionMapKey.X) self.assertEqual(actual, ['x'])
def get_field(self, ntime=2, variable_name='foo', nrow=2, ncol=2): """Create random field where mean varies with radius and std with the angle around the center of the grid. """ np.random.seed(1) row = Variable(value=np.arange(nrow) - nrow / 2., name='row', dimensions='row') col = Variable(value=np.arange(ncol) - ncol / 2., name='col', dimensions='col') grid = Grid(col, row) x, y = grid.get_value_stacked() start = dt.datetime(2000, 1, 1) delta = dt.timedelta(days=1) value_temporal = [start + i * delta for i in range(ntime)] temporal = TemporalVariable(value=value_temporal, dimensions='time', name='time') nlevel = 1 level = None nrlz = 1 realization = None value = np.random.rand(nrlz, ntime, nlevel, nrow, ncol) * np.arctan2(x, y).clip(.1) + np.hypot(x, y) variable = Variable(name=variable_name, value=value, dimensions=['realization', 'time', 'level', 'row', 'col']) field = Field(grid=grid, time=temporal, is_data=variable, level=level, realization=realization) return field
def test_from_records(self): gci = GeomCabinetIterator(path=self.path_state_boundaries) actual = Field.from_records(gci, data_model='NETCDF3_CLASSIC') desired = {'UGID': np.int32, 'ID': np.int32} for v in desired.keys(): self.assertEqual(actual[v].get_value().dtype, desired[v])
def test_time(self): units = [None, 'days since 2012-1-1'] calendar = [None, '365_day'] value = [10, 20] bounds = [[5, 15], [15, 25]] variable_type = [Variable, TemporalVariable] bounds_variable_type = [Variable, TemporalVariable] keywords = dict(units=units, calendar=calendar, variable_type=variable_type, bounds_variable_type=bounds_variable_type) for k in self.iter_product_keywords(keywords): attrs = {'units': k.units, 'calendar': k.calendar} dimension_map = {'time': {'variable': 'time', 'bounds': 'time_bnds', 'attrs': attrs}} var = k.variable_type(name='time', value=value, attrs=attrs, dimensions=['one']) bounds_var = k.bounds_variable_type(name='time_bnds', value=bounds, dimensions=['one', 'two']) f = Field(variables=[var, bounds_var], dimension_map=dimension_map) self.assertTrue(len(f.dimension_map._storage) == 1) self.assertTrue(f.time.has_bounds) self.assertIsInstance(f.time, TemporalVariable) self.assertIsInstance(f.time.bounds, TemporalVariable) self.assertEqual(f.time.value_datetime.shape, (2,)) self.assertEqual(f.time.bounds.value_datetime.shape, (2, 2)) if k.units is None: desired = constants.DEFAULT_TEMPORAL_UNITS else: desired = k.units self.assertEqual(f.time.units, desired) if k.calendar is None: desired = constants.DEFAULT_TEMPORAL_CALENDAR else: desired = k.calendar self.assertEqual(f.time.calendar, desired) self.assertEqual(f.time.bounds.calendar, desired)
def get_wrap_field(crs=None, unwrapped=True): ompi = OcgDist() ompi.create_dimension('x', 5, dist=False) ompi.create_dimension('y', 7, dist=True) ompi.create_dimension('time', size_current=4, dist=False) ompi.update_dimension_bounds() if MPI_RANK == 0: row = Variable(value=[-60, -40, -20, 0, 20, 40, 60], name='y', dimensions='y') if unwrapped: col_value = [1, 90, 180, 225, 270] else: col_value = [-170, -85, 0, 85, 170] col = Variable(value=col_value, name='x', dimensions='x') grid = Grid(col, row) value = np.zeros((4, 7, 5)) for col_idx in range(value.shape[-1]): value[:, :, col_idx] = col_idx time = TemporalVariable(name='time', value=[1, 2, 3, 4], dimensions='time') var = Variable(name='foo', value=value, dimensions=['time', 'y', 'x']) field = Field(grid=grid, is_data=var, crs=crs, time=time) else: field = None field = variable_collection_scatter(field, ompi) return field
def test_system_process_geometries(self): """Test multiple geometries with coordinate system update.""" a = 'POLYGON((-105.21347987288135073 40.21514830508475313,-104.39928495762711691 40.21514830508475313,-104.3192002118643984 39.5677966101694949,-102.37047139830508513 39.61451271186440692,-102.12354343220337682 37.51896186440677639,-105.16009004237288593 37.51896186440677639,-105.21347987288135073 40.21514830508475313))' b = 'POLYGON((-104.15235699152542281 39.02722457627118757,-103.71189088983049942 39.44099576271186436,-102.71750529661017026 39.28082627118644155,-102.35712394067796538 37.63908898305084705,-104.13900953389830306 37.63241525423728717,-104.15235699152542281 39.02722457627118757))' geom = [{'geom': wkt.loads(xx), 'properties': {'UGID': ugid}} for ugid, xx in enumerate([a, b])] grid_value = [ [[37.0, 37.0, 37.0, 37.0], [38.0, 38.0, 38.0, 38.0], [39.0, 39.0, 39.0, 39.0], [40.0, 40.0, 40.0, 40.0]], [[-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0]]] output_crs = CoordinateReferenceSystem( value={'a': 6370997, 'lon_0': -100, 'y_0': 0, 'no_defs': True, 'proj': 'laea', 'x_0': 0, 'units': 'm', 'b': 6370997, 'lat_0': 45}) x = Variable('x', grid_value[1], dimensions=['lat', 'lon']) y = Variable('y', grid_value[0], dimensions=['lat', 'lon']) grid = Grid(x, y) field = Field(grid=grid, crs=Spherical()) ops = OcgOperations(dataset=field, geom=geom, output_crs=output_crs) ret = ops.execute() expected = {0: -502052.79407259845, 1: -510391.37909706926} for field, container in ret.iter_fields(yield_container=True): self.assertAlmostEqual(field.grid.get_value_stacked().mean(), expected[container.geom.ugid.get_value()[0]])
def test_system_regridding_crs(self): """Test with coordinate systems.""" dest_crs = WGS84() grid_spherical = self.get_gridxy_global(resolution=10.0, wrapped=False, crs=Spherical()) self.assertEqual(grid_spherical.crs, Spherical()) coords = grid_spherical.get_value_stacked() data_value = self.get_exact_field_value(coords[1], coords[0]) desired = data_value.copy() data_var = Variable(name='data_src', value=data_value, dimensions=grid_spherical.dimensions) source = Field(grid=grid_spherical, is_data=data_var, crs=grid_spherical.crs) self.assertEqual(source.crs, Spherical()) destination = deepcopy(source) destination.update_crs(dest_crs) source_expanded = deepcopy(source.grid) source_expanded.expand() diff = np.abs(destination.y.get_value() - source_expanded.y.get_value()) self.assertAlmostEqual(diff.max(), 0.19231511439) for output_crs in [None, WGS84()]: ops = OcgOperations(dataset=source, regrid_destination=destination, output_crs=output_crs) ret = ops.execute() actual = ret.get_element(variable_name=data_var.name) if output_crs is None: self.assertEqual(actual.parent.crs, Spherical()) else: self.assertEqual(actual.parent.crs, WGS84()) actual = actual.get_value() diff = np.abs(actual - desired) self.assertTrue(diff.max() < 1e-5)
def add_field(self, field, container, force=False): """ Add a field to the spatial collection. :param field: The field to add. :type field: :class:`~ocgis.Field` :param container: The container geometry. A ``None`` value is allowed. :type container: :class:`~ocgis.Field` | ``None`` :param bool force: If ``True``, clobber any field names in the spatial collection. :return: """ # Assume a NoneType container if there is no geometry associated with the container. if container is not None and container.geom is not None: ugid = container.geom.ugid.get_value()[0] if ugid not in self.children: self.children[ugid] = container else: # We want to use the reference to the container in the collection. container = self.children[ugid] container.add_child(field, force=force) else: if None not in self.children: self.children[None] = Field() container = self.children[None] container.add_child(field, force=force)
def test_create_dimension_map_2d_spatial_coordinates(self): grid = create_gridxy_global() grid.expand() path = self.get_temporary_file_path('foo.nc') f = Field(grid=grid) f.write(path) rd = RequestDataset(path) field = rd.get() sub = field.get_field_slice({'y': 10, 'x': 5}) self.assertEqual(sub.grid.x.shape, (1, 1)) actual = f.dimension_map.get_dimension(DimensionMapKey.Y) self.assertEqual(actual, ['y']) actual = f.dimension_map.get_dimension(DimensionMapKey.X) self.assertEqual(actual, ['x'])
def test_properties(self): # Test an empty properties dictionary is created if there are no data variables on the container/parent field. gvar = GeometryVariable(name='empty', value=[Point(1, 2)], ugid=20, dimensions='ngeom') info = Variable(name='info') gvar.parent.add_variable(info) sc = SpatialCollection() sc.add_field(Field(name='what'), gvar.parent) self.assertEqual(len(sc.properties), 1)
def test_system_dimension_map_formatting(self): """Test any formatting of the incoming dimension map by the field.""" dmap = {'time': {'variable': 'time'}} time = TemporalVariable(name='time', value=[1, 2, 3], dimensions='the_time') field = Field(time=time, dimension_map=dmap) actual = field.dimension_map.get_dimension('time') self.assertEqual(actual, ['the_time'])
def test_crs(self): """Test overloading by geometry and grid.""" field = Field() self.assertIsNone(field.crs) geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical()) field = Field(geom=geom) self.assertEqual(field.crs, geom.crs) grid = self.get_gridxy_global(crs=Spherical()) field = Field(grid=grid) self.assertEqual(field.crs, grid.crs) grid = self.get_gridxy_global(crs=Spherical()) # Grid and field coordinate systems do not match. with self.assertRaises(ValueError): Field(grid=grid, crs=WGS84()) geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical()) with self.assertRaises(ValueError): Field(geom=geom, crs=WGS84()) geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g') grid = self.get_gridxy_global() field = Field(geom=geom, grid=grid, crs=WGS84()) self.assertEqual(field.crs, WGS84()) self.assertEqual(field.geom.crs, WGS84()) self.assertEqual(field.grid.crs, WGS84()) g = self.get_gridxy_global() f = Field(grid=g, crs=Spherical()) self.assertIn('standard_name', f.grid.x.attrs) self.assertIn('standard_name', f.grid.y.attrs)
def test_update_crs(self): # Test copying allows the CRS to be updated on the copy w/out changing the source CRS. desired = Spherical() gvar = GeometryVariable(value=[Point(1, 2)], name='geom', dimensions='geom') field = Field(crs=desired, geom=gvar) cfield = field.copy() self.assertEqual(cfield.crs, desired) new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326) cfield.update_crs(new_crs) self.assertEqual(field.crs, desired) # Test geometry crs update is called. mfield = Mock(Field) mfield.is_empty = False mfield.dimension_map = Mock(DimensionMap) mfield.grid = Mock(Grid) mfield.geom = Mock(GeometryVariable) mcrs = Mock(Spherical) Field.update_crs(mfield, mcrs) mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs) mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs) from_crs = Mock(WGS84) mfield.grid.update_crs.reset_mock() mfield.geom.update_crs.reset_mock() Field.update_crs(mfield, mcrs, from_crs=from_crs) mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=from_crs) mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
def test_system_with_time_data(self): """Test writing data with a time dimension.""" path = self.get_temporary_file_path('what.shp') t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time') geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='time') field = Field(variables=[t, geom], dimension_map={'time': {'variable': 'time'}, 'geom': {'variable': 'geom'}}) field.write(path, iter_kwargs={'variable': 'time'}, driver=DriverVector) rd = RequestDataset(uri=path) field2 = rd.get() # netcdftime worthlessness poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'], ['1-01-02 12:00:00', '1-01-03 12:00:00']] actual = field2['TIME'].get_value().tolist() res = [p == actual for p in poss] self.assertTrue(any(res))
def test_system_masking(self): """Test how the mask is handled by the NetCDF python library.""" var = Variable(name='foo', value=[1, 2, 3], dimensions='three') self.assertIsNone(var.get_mask()) path = self.get_temporary_file_path('foo.nc') var.parent.write(path) invar = Field.read(path)['foo'] self.assertIsNone(invar.get_mask())
def create_regridding_field(grid, name_variable): col_shape = grid.shape[1] row_shape = grid.shape[0] value = np.ones(col_shape * row_shape, dtype=float).reshape( grid.shape) * 15. variable = Variable(name=name_variable, value=value, dimensions=grid.dimensions) field = Field(is_data=variable, grid=grid, crs=Spherical()) return field
def get_field(self): gridxy = self.get_gridxy(with_xy_bounds=True) coords_stacked = gridxy.get_value_stacked() exact = self.get_exact_field_value(coords_stacked[1], coords_stacked[0]) exact = Variable(name='exact1', value=exact, dimensions=gridxy.dimensions) dimension_map = {'x': {'variable': 'x', 'bounds': 'xbounds'}, 'y': {'variable': 'y', 'bounds': 'ybounds'}} field1 = Field.from_variable_collection(gridxy.parent, dimension_map=dimension_map) field1.add_variable(exact) field1.set_name('exact1') field1.append_to_tags(TagName.DATA_VARIABLES, 'exact1') return field1
def test_set_x(self): f = Field() var = Variable('x', value=[1, 2], dimensions='xdim') f.set_x(var, 'xdim') var2 = Variable('x2', value=[3, 4], dimensions='xdim2') f.set_x(var2, 'xdim2') self.assertNotIn(var.name, f) f.set_x(None, None) self.assertEqual(len(f), 0) self.assertIsNone(f.x)
def get_subset_field(self): crs = self.crs geoms = self.geoms gridcode = Variable('gridcode', [110101, 12103], dimensions='ngeom') description = Variable('description', ['high point', 'low point'], dimensions='ngeom') dimension_map = {'geom': {'variable': 'geoms', DimensionMapKey.DIMENSION: ['ngeom']}, 'crs': {'variable': crs.name}} poi = Field(variables=[geoms, gridcode, description], dimension_map=dimension_map, is_data=[gridcode, description]) geoms.set_ugid(gridcode) return poi
def _run_(): env.SUPPRESS_WARNINGS = False ocgis_lh.configure(to_stream=True) records = [{ 'geom': Point(1, 2), 'properties': { 'a_list': [1, 2, 3] } }] actual = Field.from_records(records) self.assertNotIn("a_list", actual.keys()) env.SUPPRESS_WARNINGS = True
def read_from_collection(target, request_dataset, parent=None, name=None, source_name=constants.UNINITIALIZED, uid=None): # Allow an empty variable renaming map. This occurs when there are no visible data variables to the metadata # parser. try: rename_variable_map = request_dataset.rename_variable_map except NoDataVariablesFound: rename_variable_map = {} ret = Field(attrs=get_netcdf_attributes(target), parent=parent, name=name, source_name=source_name, uid=uid) pred = request_dataset.predicate for varname, ncvar in target.variables.items(): if pred is not None and not pred(varname): continue source_name = varname name = rename_variable_map.get(varname, varname) sv = SourcedVariable(name=name, request_dataset=request_dataset, parent=ret, source_name=source_name) ret[name] = sv for group_name, ncgroup in list(target.groups.items()): child = read_from_collection(ncgroup, request_dataset, parent=ret, name=group_name, uid=uid) ret.add_child(child) return ret
def test_write_parallel(self): """Test writing by selective rank.""" if MPI_SIZE != 3 and MPI_SIZE != 1: raise SkipTest('MPI_SIZE != 1 or 3') ranks = list(range(MPI_SIZE)) for base_rank in ranks: for driver in [ DriverCSV, DriverVector, DriverNetcdf ]: if MPI_RANK == 0: path = self.get_temporary_file_path('{}-{}.{}'.format(driver.key, base_rank, driver.common_extension)) else: path = None path = MPI_COMM.bcast(path) with vm.scoped('field write by rank', [base_rank]): if not vm.is_null: geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='geom') data = Variable(name='data', value=[10, 20], dimensions='geom') field = Field(geom=geom) field.add_variable(data, is_data=True) self.assertFalse(os.path.isdir(path)) field.write(path, driver=driver) self.assertFalse(os.path.isdir(path)) rd = RequestDataset(path, driver=driver) in_field = rd.get() self.assertEqual(in_field['data'].dimensions[0].size, 2) MPI_COMM.Barrier() MPI_COMM.Barrier()
def test_system_crs_and_grid_abstraction(self): f = Field(grid_abstraction='point') grid = self.get_gridxy(with_xy_bounds=True) f.add_variable(grid.x) crs = CoordinateReferenceSystem(epsg=2136, name='location') f.add_variable(crs) self.assertIsNone(f.crs) f.dimension_map.set_crs(crs) f.dimension_map.set_variable('x', grid.x) f.dimension_map.set_variable('y', grid.y) self.assertEqual(f.grid.crs, crs) f.set_geom(f.grid.get_abstraction_geometry()) self.assertEqual(f.grid.abstraction, 'point') self.assertEqual(f.geom.geom_type, 'Point')
def test_get_field_write_target(self): # Test coordinate system names are added to attributes of dimensioned variables. x = Variable('x', dimensions='x', value=[1]) y = Variable('y', dimensions='y', value=[2]) t = Variable('t', dimensions='t', value=[3]) crs = WGS84() d = Variable('data', dimensions=['t', 'y', 'x'], value=[[[1]]]) grid = Grid(x, y) field = Field(grid=grid, time=t, crs=crs) field.add_variable(d, is_data=True) target = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(target[d.name].attrs['grid_mapping'], crs.name) self.assertEqual(field.x.units, 'degrees_east') # Test bounds units are removed when writing. x = Variable(name='x', value=[1, 2, 3], dtype=float, dimensions='xdim', units='hours') y = Variable(name='y', value=[1, 2, 3], dtype=float, dimensions='ydim', units='hours') grid = Grid(x, y) grid.set_extrapolated_bounds('x_bounds', 'y_bounds', 'bounds') self.assertEqual(x.bounds.units, x.units) self.assertEqual(y.bounds.units, y.units) field = Field(grid=grid) actual = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(x.bounds.units, x.units) self.assertNumpyMayShareMemory(actual[x.name].get_value(), field[x.name].get_value()) self.assertIsNone(actual[x.name].bounds.units) self.assertIsNone(actual[y.name].bounds.units) self.assertEqual(x.bounds.units, x.units) self.assertEqual(y.bounds.units, y.units) # Test actual coordinate system is triggered. field = Field() src = CFSpherical() dst = WGS84() field.set_crs(src) self.assertEqual(field.crs, src) self.assertIsNone(env.COORDSYS_ACTUAL) env.COORDSYS_ACTUAL = dst actual = DriverNetcdfCF._get_field_write_target_(field) self.assertEqual(actual.crs, dst) self.assertEqual(field.crs, src) self.assertNotIn(src.name, actual) self.assertIn(dst.name, actual)
def _update_aggregation_wrapping_crs_(obj, alias, sfield, subset_sdim, subset_ugid): raise_if_empty(sfield) ocgis_lh('entering _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias, ugid=subset_ugid, level=logging.DEBUG) # Aggregate if requested. if obj.ops.aggregate: ocgis_lh('aggregate requested in _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias, ugid=subset_ugid, level=logging.DEBUG) # There may be no geometries if we are working with a gridded dataset. Load the geometries if this is the case. sfield.set_abstraction_geom() ocgis_lh('after sfield.set_abstraction_geom in _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias, ugid=subset_ugid, level=logging.DEBUG) # Union the geometries and spatially average the data variables. # with vm.scoped(vm.get_live_ranks_from_object(sfield)): sfield = sfield.geom.get_unioned(spatial_average=sfield.data_variables) ocgis_lh('after sfield.geom.get_unioned in _update_aggregation_wrapping_crs_', obj._subset_log, alias=alias, ugid=subset_ugid, level=logging.DEBUG) # None is returned for the non-root process. Check we are in parallel and create an empty field. if sfield is None: if vm.size == 1: raise ValueError('None should not be returned from get_unioned if running on a single processor.') else: sfield = Field(is_empty=True) else: sfield = sfield.parent vm.create_subcomm_by_emptyable(SubcommName.SPATIAL_AVERAGE, sfield, is_current=True, clobber=True) if not vm.is_null and subset_sdim is not None and subset_sdim.geom is not None: # Add the unique geometry identifier variable. This should match the selection geometry's identifier. new_gid_variable_kwargs = dict(name=HeaderName.ID_GEOMETRY, value=subset_sdim.geom.ugid.get_value(), dimensions=sfield.geom.dimensions) dm = get_data_model(obj.ops) new_gid_variable = create_typed_variable_from_data_model('int', data_model=dm, **new_gid_variable_kwargs) sfield.geom.set_ugid(new_gid_variable) if vm.is_null: ocgis_lh(msg='null communicator following spatial average. returning.', logger=obj._subset_log, level=logging.DEBUG) return sfield raise_if_empty(sfield) ocgis_lh(msg='before wrapped_state in _update_aggregation_wrapping_crs_', logger=obj._subset_log, level=logging.DEBUG) try: wrapped_state = sfield.wrapped_state except WrappedStateEvalTargetMissing: # If there is no target for wrapping evaluation, then consider this unknown. wrapped_state = WrappedState.UNKNOWN ocgis_lh(msg='after wrapped_state in _update_aggregation_wrapping_crs_', logger=obj._subset_log, level=logging.DEBUG) # Wrap the returned data. if not env.OPTIMIZE_FOR_CALC and not sfield.is_empty: if wrapped_state == WrappedState.UNWRAPPED: ocgis_lh('wrap target is empty: {}'.format(sfield.is_empty), obj._subset_log, level=logging.DEBUG) # There may be no geometries if we are working with a gridded dataset. Load the geometries if this # is the case. sfield.set_abstraction_geom() if obj.ops.output_format in constants.VECTOR_OUTPUT_FORMATS and obj.ops.vector_wrap: ocgis_lh('wrapping output geometries', obj._subset_log, alias=alias, ugid=subset_ugid, level=logging.DEBUG) # Deepcopy geometries before wrapping as wrapping will be performed inplace. The original field may # need to be reused for additional subsets. geom = sfield.geom copied_geom = geom.get_value().copy() geom.set_value(copied_geom) # Some grids do not play nicely with wrapping. Bounds may be less than zero for an unwrapped grid. # Force wrapping if it is requested. Normally, when force is false there is a pass-through that will # leave the data untouched. geom.wrap(force=True) ocgis_lh('finished wrapping output geometries', obj._subset_log, alias=alias, ugid=subset_ugid, level=logging.DEBUG) # Transform back to rotated pole if necessary. original_rotated_pole_crs = obj._backtransform.get(constants.BackTransform.ROTATED_POLE) if original_rotated_pole_crs is not None: if not isinstance(obj.ops.output_crs, (Spherical, WGS84)): sfield.update_crs(original_rotated_pole_crs) # Update the coordinate system of the data output. if obj.ops.output_crs is not None: # If the geometry is not none, it may need to be projected to match the output coordinate system. if subset_sdim is not None and subset_sdim.crs != obj.ops.output_crs: subset_sdim.update_crs(obj.ops.output_crs) # Update the subsetted field's coordinate system. sfield = sfield.copy() sfield.update_crs(obj.ops.output_crs) # Wrap or unwrap the data if the coordinate system permits. _update_wrapping_(obj, sfield) ocgis_lh('leaving _update_aggregation_wrapping_crs_', obj._subset_log, level=logging.DEBUG) return sfield
def write_chunks(self): """ Write grid subsets to netCDF files using the provided filename templates. This will also generate ESMF regridding weights for each subset if requested. """ src_filenames = [] dst_filenames = [] wgt_filenames = [] dst_slices = [] src_slices = [] index_path = self.create_full_path_from_template('index_file') # nzeros = len(str(reduce(lambda x, y: x * y, self.nchunks_dst))) ctr = 1 ocgis_lh(logger='grid_chunker', msg='starting self.iter_src_grid_subsets', level=logging.DEBUG) for sub_src, src_slc, sub_dst, dst_slc in self.iter_src_grid_subsets(yield_dst=True): ocgis_lh(logger='grid_chunker', msg='finished iteration {} for self.iter_src_grid_subsets'.format(ctr), level=logging.DEBUG) src_path = self.create_full_path_from_template('src_template', index=ctr) dst_path = self.create_full_path_from_template('dst_template', index=ctr) wgt_path = self.create_full_path_from_template('wgt_template', index=ctr) src_filenames.append(os.path.split(src_path)[1]) dst_filenames.append(os.path.split(dst_path)[1]) wgt_filenames.append(wgt_path) dst_slices.append(dst_slc) src_slices.append(src_slc) # Only write destinations if an iterator is not provided. if self.iter_dst is None: zip_args = [[sub_src, sub_dst], [src_path, dst_path]] else: zip_args = [[sub_src], [src_path]] cc = 1 for target, path in zip(*zip_args): with vm.scoped_by_emptyable('field.write' + str(cc), target): if not vm.is_null: ocgis_lh(logger='grid_chunker', msg='write_chunks:writing: {}'.format(path), level=logging.DEBUG) field = Field(grid=target) field.write(path) ocgis_lh(logger='grid_chunker', msg='write_chunks:finished writing: {}'.format(path), level=logging.DEBUG) cc += 1 # Increment the counter outside of the loop to avoid counting empty subsets. ctr += 1 # Generate an ESMF weights file if requested and at least one rank has data on it. if self.genweights and len(vm.get_live_ranks_from_object(sub_src)) > 0: vm.barrier() self.write_esmf_weights(src_path, dst_path, wgt_path, src_grid=sub_src, dst_grid=sub_dst) vm.barrier() # Global shapes require a VM global scope to collect. src_global_shape = global_grid_shape(self.src_grid) dst_global_shape = global_grid_shape(self.dst_grid) # Gather and collapse source slices as some may be empty and we write on rank 0. gathered_src_grid_slice = vm.gather(src_slices) if vm.rank == 0: len_src_slices = len(src_slices) new_src_grid_slice = [None] * len_src_slices for idx in range(len_src_slices): for rank_src_grid_slice in gathered_src_grid_slice: if rank_src_grid_slice[idx] is not None: new_src_grid_slice[idx] = rank_src_grid_slice[idx] break src_slices = new_src_grid_slice with vm.scoped('index write', [0]): if not vm.is_null: dim = Dimension('nfiles', len(src_filenames)) vname = ['source_filename', 'destination_filename', 'weights_filename'] values = [src_filenames, dst_filenames, wgt_filenames] grid_chunker_destination = GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE attrs = [{'esmf_role': 'grid_chunker_source'}, {'esmf_role': grid_chunker_destination}, {'esmf_role': 'grid_chunker_weights'}] vc = VariableCollection() grid_chunker_index = GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE vidx = Variable(name=grid_chunker_index) vidx.attrs['esmf_role'] = grid_chunker_index vidx.attrs['grid_chunker_source'] = 'source_filename' vidx.attrs[GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE] = 'destination_filename' vidx.attrs['grid_chunker_weights'] = 'weights_filename' vidx.attrs[GridChunkerConstants.IndexFile.NAME_SRC_GRID_SHAPE] = src_global_shape vidx.attrs[GridChunkerConstants.IndexFile.NAME_DST_GRID_SHAPE] = dst_global_shape vc.add_variable(vidx) for idx in range(len(vname)): v = Variable(name=vname[idx], dimensions=dim, dtype=str, value=values[idx], attrs=attrs[idx]) vc.add_variable(v) bounds_dimension = Dimension(name='bounds', size=2) # TODO: This needs to work with four dimensions. # Source ----------------------------------------------------------------------------------------------- self.src_grid._gc_create_index_bounds_(RegriddingRole.SOURCE, vidx, vc, src_slices, dim, bounds_dimension) # Destination ------------------------------------------------------------------------------------------ self.dst_grid._gc_create_index_bounds_(RegriddingRole.DESTINATION, vidx, vc, dst_slices, dim, bounds_dimension) vc.write(index_path) vm.barrier()
def create_field(self, *args, **kwargs): """ Create a field object. In general, this should not be overloaded by subclasses. :keyword bool format_time: ``(=True)`` If ``False``, do not convert numeric times to Python date objects. :keyword str grid_abstraction: ``(='auto')`` If provided, use this grid abstraction. :keyword raw_field: ``(=None)`` If provided, modify this field instead. :type raw_field: None | :class:`~ocgis.Field` :param kwargs: Additional keyword arguments to :meth:`~ocgis.driver.base.AbstractDriver.create_raw_field`. :return: :class:`ocgis.Field` """ kwargs = kwargs.copy() raw_field = kwargs.pop('raw_field', None) format_time = kwargs.pop(KeywordArgument.FORMAT_TIME, True) grid_abstraction = kwargs.pop(KeywordArgument.GRID_ABSTRACTION, self.rd.grid_abstraction) grid_is_isomorphic = kwargs.pop('grid_is_isomorphic', self.rd.grid_is_isomorphic) if raw_field is None: # Get the raw variable collection from source. new_kwargs = kwargs.copy() new_kwargs['source_name'] = None raw_field = self.create_raw_field(*args, **new_kwargs) # Get the appropriate metadata for the collection. group_metadata = self.get_group_metadata(raw_field.group, self.metadata_source) # Always pull the dimension map from the request dataset. This allows it to be overloaded. dimension_map = self.get_group_metadata(raw_field.group, self.rd.dimension_map) # Modify the coordinate system variable. If it is overloaded on the request dataset, then the variable # collection needs to be updated to hold the variable and any alternative coordinate systems needs to be # removed. to_remove = None to_add = None crs = self.get_crs(group_metadata) if self.rd._has_assigned_coordinate_system: to_add = self.rd._crs if crs is not None: to_remove = crs.name else: if self.rd._crs is not None and self.rd._crs != 'auto': to_add = self.rd._crs if crs is not None: to_remove = crs.name elif crs is not None: to_add = crs if to_remove is not None: raw_field.pop(to_remove, None) if to_add is not None: raw_field.add_variable(to_add, force=True) # Overload the dimension map with the CRS. if to_add is not None: # dimension_map[DimensionMapKey.CRS][DimensionMapKey.VARIABLE] = to_add.name dimension_map.set_crs(to_add.name) # Remove the mask variable if present in the raw dimension map and the source dimension map is set to None. if self.rd.dimension_map.get_spatial_mask() is None and self.dimension_map_raw.get_spatial_mask() is not None: raw_field.pop(self.dimension_map_raw.get_spatial_mask()) # Convert the raw variable collection to a field. # TODO: Identify a way to remove this code block; field should be appropriately initialized; format_time and grid_abstraction are part of a dimension map. kwargs[KeywordArgument.DIMENSION_MAP] = dimension_map kwargs[KeywordArgument.FORMAT_TIME] = format_time if grid_abstraction != 'auto': kwargs[KeywordArgument.GRID_ABSTRACTION] = grid_abstraction if grid_is_isomorphic != 'auto': kwargs['grid_is_isomorphic'] = grid_is_isomorphic field = Field.from_variable_collection(raw_field, *args, **kwargs) # If this is a source grid for regridding, ensure the flag is updated. field.regrid_source = self.rd.regrid_source # Update the assigned coordinate system flag. field._has_assigned_coordinate_system = self.rd._has_assigned_coordinate_system # Apply any requested subsets. if self.rd.time_range is not None: field = field.time.get_between(*self.rd.time_range).parent if self.rd.time_region is not None: field = field.time.get_time_region(self.rd.time_region).parent if self.rd.time_subset_func is not None: field = field.time.get_subset_by_function(self.rd.time_subset_func).parent if self.rd.level_range is not None: field = field.level.get_between(*self.rd.level_range).parent # These variables have all the dimensions needed for a data classification. Use overloaded values from the # request dataset if they are provided. try: data_variable_names = list(get_variable_names(self.rd.rename_variable)) except NoDataVariablesFound: # It is okay to have no data variables in a field. data_variable_names = [] pass for dvn in data_variable_names: field.append_to_tags(TagName.DATA_VARIABLES, dvn, create=True) # Load child fields. for child in list(field.children.values()): kwargs['raw_field'] = child field.children[child.name] = self.create_field(*args, **kwargs) return field
def test_write(self): # Test writing a basic grid. path = self.get_temporary_file_path('foo.nc') x = Variable(name='x', value=[1, 2], dimensions='x') y = Variable(name='y', value=[3, 4, 5, 6, 7], dimensions='y') dmap = {'x': {'variable': 'x'}, 'y': {'variable': 'y'}} field = Field(variables=[x, y], dimension_map=dmap) desired_value_stacked = field.grid.get_value_stacked() self.assertEqual(field.grid.parent['x'].get_value().shape, (2,)) self.assertTrue(field.grid.is_vectorized) field.write(path) out_field = RequestDataset(path).get() self.assertTrue(out_field.grid.is_vectorized) actual_value_stacked = out_field.grid.get_value_stacked() self.assertNumpyAll(actual_value_stacked, desired_value_stacked) # Test another grid. grid = self.get_gridxy(crs=WGS84()) self.assertTrue(grid.is_vectorized) field = Field(grid=grid) self.assertTrue(field.grid.is_vectorized) path = self.get_temporary_file_path('out.nc') with self.nc_scope(path, 'w') as ds: field.write(ds) self.assertTrue(field.grid.is_vectorized) with self.nc_scope(path) as ds: if atleast_ncver("1.4"): actual = grid.x.mv() else: actual = grid.x.v() self.assertNumpyAll(ds.variables[grid.x.name][:], actual) var = ds.variables[grid.y.name] if atleast_ncver("1.4"): actual = grid.y.mv() else: actual = grid.y.v() self.assertNumpyAll(var[:], actual) self.assertEqual(var.axis, 'Y') self.assertIn(grid.crs.name, ds.variables) # Test with 2-d x and y arrays. grid = self.get_gridxy(with_2d_variables=True) field = Field(grid=grid) path = self.get_temporary_file_path('out.nc') field.grid.set_extrapolated_bounds('xbounds', 'ybounds', 'bounds') with self.nc_scope(path, 'w') as ds: field.write(ds) # self.ncdump(path) with self.nc_scope(path) as ds: var = ds.variables['y'] if atleast_ncver("1.4"): actual = grid.y.mv() else: actual = grid.y.v() self.assertNumpyAll(var[:], actual) # Test writing a vectorized grid with corners. grid = self.get_gridxy() field = Field(grid=grid) self.assertIsNotNone(field.grid.dimensions) self.assertFalse(field.grid.has_bounds) field.grid.set_extrapolated_bounds('xbnds', 'ybnds', 'corners') self.assertTrue(field.grid.is_vectorized) path = self.get_temporary_file_path('out.nc') with self.nc_scope(path, 'w') as ds: field.write(ds) # self.ncdump(path) with self.nc_scope(path, 'r') as ds: self.assertEqual(['ydim'], [d for d in ds.variables['y'].dimensions]) self.assertEqual(['xdim'], [d for d in ds.variables['x'].dimensions])