def test_crs(self): """Test overloading by geometry and grid.""" field = Field() self.assertIsNone(field.crs) geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical()) field = Field(geom=geom) self.assertEqual(field.crs, geom.crs) grid = self.get_gridxy_global(crs=Spherical()) field = Field(grid=grid) self.assertEqual(field.crs, grid.crs) grid = self.get_gridxy_global(crs=Spherical()) # Grid and field coordinate systems do not match. with self.assertRaises(ValueError): Field(grid=grid, crs=WGS84()) geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical()) with self.assertRaises(ValueError): Field(geom=geom, crs=WGS84()) geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g') grid = self.get_gridxy_global() field = Field(geom=geom, grid=grid, crs=WGS84()) self.assertEqual(field.crs, WGS84()) self.assertEqual(field.geom.crs, WGS84()) self.assertEqual(field.grid.crs, WGS84()) g = self.get_gridxy_global() f = Field(grid=g, crs=Spherical()) self.assertIn('standard_name', f.grid.x.attrs) self.assertIn('standard_name', f.grid.y.attrs)
def test_get_buffered_geometry(self): proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs' buffer_crs_list = [None, CoordinateReferenceSystem(proj4=proj4)] poly = make_poly((36, 44), (-104, -95)) for buffer_crs in buffer_crs_list: gvar = GeometryVariable(value=poly, name='geoms', dimensions='dim', crs=WGS84()) self.assertEqual(gvar.crs, WGS84()) if buffer_crs is None: buffer_value = 1 else: buffer_value = 10 ret = SpatialSubsetOperation._get_buffered_geometry_(gvar, buffer_value, buffer_crs=buffer_crs) ref = ret.get_value()[0] if buffer_crs is None: self.assertEqual(ref.bounds, (-105.0, 35.0, -94.0, 45.0)) else: self.assertNumpyAllClose(np.array(ref.bounds), np.array( (-104.00013263459613, 35.9999147913708, -94.99986736540386, 44.00008450528758))) self.assertEqual(gvar.crs, ret.crs) # check deepcopy ret.get_value()[0] = make_poly((1, 2), (3, 4)) ref_buffered = ret.get_value()[0] ref_original = gvar.get_value()[0] with self.assertRaises(AssertionError): self.assertNumpyAllClose(np.array(ref_buffered.bounds), np.array(ref_original.bounds))
def test_get_unioned_spatial_average_parallel(self): if MPI_SIZE != 8: raise SkipTest('MPI_SIZE != 8') dist = OcgDist() geom_count = dist.create_dimension('geom_count', size=8, dist=True) time_count = dist.create_dimension('time', size=3) dist.update_dimension_bounds() if not geom_count.is_empty: gvar = GeometryVariable(value=[Point(1.0, 1.0).buffer(MPI_RANK + 2)] * len(geom_count), dimensions=geom_count) value = np.zeros((len(time_count), len(geom_count)), dtype=float) for ii in range(value.shape[0]): value[ii] = [MPI_RANK + 1 + ii + 1] * len(geom_count) data = Variable(name='data', value=value, dtype=float, dimensions=[time_count, geom_count]) else: gvar = GeometryVariable(dimensions=geom_count) data = Variable(name='data', dimensions=[time_count, geom_count], dtype=float) gvar.parent.add_variable(data) self.assertTrue(gvar.is_empty == data.is_empty == gvar.parent.is_empty) with vm.scoped_by_emptyable('union', gvar): if vm.is_null: unioned = None else: unioned = gvar.get_unioned(spatial_average='data') if unioned is not None: self.assertIsInstance(unioned, GeometryVariable) actual = unioned.parent[data.name] self.assertAlmostEqual(actual.get_value().max(), 5.5466666666666677) else: self.assertIsNone(unioned)
def test_prepare_geometry(self): for geometry_record in self.get_subset_geometries(): for ss, k in self: gvar = GeometryVariable(value=geometry_record['geom'], dimensions='dim', crs=WGS84()) self.assertIsNotNone(gvar.crs) prepared = ss._prepare_geometry_(gvar) self.assertNotEqual(gvar.get_value()[0].bounds, prepared.get_value()[0].bounds) self.assertFalse( np.may_share_memory(gvar.get_value(), prepared.get_value())) try: self.assertEqual(prepared.crs, ss.field.crs) except AssertionError: # Rotated pole on the fields become spherical. self.assertEqual(prepared.crs, CFSpherical()) self.assertIsInstance(ss.field.crs, CFRotatedPole) # Test nebraska against an unwrapped dataset. nebraska = GeometryVariable(value=self.nebraska['geom'], dimensions='d', crs=WGS84()) field = self.test_data.get_rd('cancm4_tas').get() ss = SpatialSubsetOperation(field) prepared = ss._prepare_geometry_(nebraska) self.assertEqual(prepared.wrapped_state, WrappedState.UNWRAPPED)
def test_init(self): field = self.get_ocgfield() self.assertIsInstance(field, Field) # Test unique identifier. field = Field(uid=4) self.assertEqual(field.uid, 4) # Test with a coordinate system and geometry. desired_crs = WGS84() geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='geom') field = Field(crs=desired_crs, geom=geom) self.assertEqual(field.crs, desired_crs) # Test dimension names are automatically added to dimension map. g = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='the_geom_dim') f = Field(geom=g) actual = f.dimension_map.get_dimension(DimensionMapKey.GEOM) self.assertEqual(actual, ['the_geom_dim']) # Test dimension map does not have any entries at initialization. actual = Field() desired = actual.dimension_map.as_dict() self.assertEqual(len(desired), 0)
def test_update_crs_to_cartesian(self): """Test a spherical to cartesian CRS update.""" bbox = box(-170., 40., 150., 80.) original_bounds = deepcopy(bbox.bounds) geom = GeometryVariable(name='geom', value=[bbox], dimensions='geom', crs=Spherical()) other_crs = Cartesian() geom.update_crs(other_crs) actual = geom.get_value()[0].bounds desired = (-0.7544065067354889, -0.13302222155948895, -0.15038373318043535, 0.38302222155948895) self.assertNumpyAllClose(np.array(actual), np.array(desired)) self.assertIsInstance(geom.crs, Cartesian) other_crs = Spherical() geom.update_crs(other_crs) self.assertEqual(geom.crs, Spherical()) actual = geom.get_value()[0].bounds self.assertNumpyAllClose(np.array(original_bounds), np.array(actual)) # Test data may not be wrapped. bbox = box(0, 40, 270, 80) geom = GeometryVariable(name='geom', value=[bbox], dimensions='geom', crs=Spherical()) other_crs = Cartesian() with self.assertRaises(ValueError): geom.update_crs(other_crs)
def test_init(self): # Test empty. gvar = GeometryVariable() self.assertEqual(gvar.dtype, object) gvar = self.get_geometryvariable() self.assertIsInstance(gvar.get_masked_value(), MaskedArray) self.assertEqual(gvar.ndim, 1) # Test passing a "crs". gvar = self.get_geometryvariable(crs=WGS84(), name='my_geom', dimensions='ngeom') self.assertEqual(gvar.crs, WGS84()) # Test using lines. line1 = LineString([(0, 0), (1, 1)]) line2 = LineString([(1, 1), (2, 2)]) gvar = GeometryVariable(value=[line1, line2], dimensions='two') self.assertTrue(gvar.get_value()[1].almost_equals(line2)) self.assertEqual(gvar.geom_type, line1.geom_type) lines = MultiLineString([line1, line2]) lines2 = [lines, lines] for actual in [lines, lines2, lines]: gvar2 = GeometryVariable(value=actual, dimensions='ngeom') self.assertTrue(gvar2.get_value()[0].almost_equals(lines)) self.assertEqual(gvar2.geom_type, lines.geom_type) self.assertTrue(gvar2.shape[0] > 0) self.assertIsNone(gvar2.get_mask())
def test_deepcopy(self): gvar = GeometryVariable(value=Point(1, 2), crs=Spherical(), dimensions='d') self.assertEqual(gvar.crs, Spherical()) d = gvar.deepcopy() d.crs = WGS84() self.assertEqual(gvar.crs, Spherical()) self.assertFalse(np.may_share_memory(gvar.get_value(), d.get_value())) self.assertIsNotNone(d.crs)
def test_wrap(self): geom = box(195, -40, 225, -30) gvar = GeometryVariable(name='geoms', value=geom, crs=Spherical(), dimensions='geoms') gvar.wrap() self.assertEqual(gvar.get_value()[0].bounds, (-165.0, -40.0, -135.0, -30.0))
def test_get_mask_from_intersects(self): poly = wkt.loads( 'POLYGON((-98.26574367088608142 40.19952531645570559,-98.71764240506330168 39.54825949367089066,-99.26257911392406186 39.16281645569620906,-99.43536392405064817 38.64446202531645724,-98.78409810126584034 38.33876582278481493,-98.23916139240508016 37.71408227848101546,-97.77397151898735217 37.67420886075949937,-97.62776898734178133 38.15268987341772799,-98.39865506329114453 38.52484177215190186,-98.23916139240508016 39.33560126582278826,-97.73409810126582897 39.58813291139241386,-97.52143987341773368 40.27927215189873777,-97.52143987341773368 40.27927215189873777,-98.26574367088608142 40.19952531645570559))' ) desired_mask = np.array([[True, True, False, True], [True, False, True, True], [True, True, False, True]]) dist = OcgDist() xdim = dist.create_dimension('x', 4, dist=True) ydim = dist.create_dimension('y', 3) dist.create_dimension('bounds', 2) dist.update_dimension_bounds() if MPI_RANK == 0: x = self.get_variable_x() y = self.get_variable_y() grid = Grid(x=x, y=y, abstraction='point', crs=WGS84()) pa = get_geometry_variable(grid) else: pa = None pa = variable_scatter(pa, dist) vm.create_subcomm_by_emptyable('test_get_mask_from_intersects', pa, is_current=True) if vm.is_null: self.assertTrue(pa.is_empty) return usi = [False] if env.USE_SPATIAL_INDEX: usi.append(True) keywords = dict(use_spatial_index=usi) for k in self.iter_product_keywords(keywords): ret = pa.get_mask_from_intersects( poly, use_spatial_index=k.use_spatial_index) desired_mask_local = desired_mask[slice(*ydim.bounds_local), slice(*xdim.bounds_local)] if MPI_RANK > 1: self.assertIsNone(ret) else: self.assertNumpyAll(desired_mask_local, ret) # This does not test a parallel operation. if MPI_RANK == 0: # Test pre-masked values in geometry are okay for intersects operation. value = [Point(1, 1), Point(2, 2), Point(3, 3)] value = np.ma.array(value, mask=[False, True, False], dtype=object) pa2 = GeometryVariable(value=value, dimensions='ngeom') b = box(0, 0, 5, 5) res = pa2.get_mask_from_intersects( b, use_spatial_index=k.use_spatial_index) self.assertNumpyAll(res, value.mask)
def test_as_shapely(self): coords = (12, 3, 15, 4) bbox = box(*coords) gvar = GeometryVariable(value=bbox, is_bbox=True, dimensions='geom', crs=Spherical()) geom = gvar.as_shapely() self.assertEqual(geom.bounds, coords)
def test_prepare(self): coords = (-10.0, 40.0, 50.0, 50.0) bbox = box(*coords) gvar = GeometryVariable(value=bbox, dimensions='geom', crs=Spherical(), is_bbox=True, wrapped_state=WrappedState.UNWRAPPED) for _ in range(3): prepared = gvar.prepare() self.assertNotEqual(id(prepared), id(gvar)) actual = [] desired = [(-10.0, 40.0, 50.0, 50.0), (350.0, 40.0, 370.0, 50.0)] for g in prepared.get_value().flatten()[0]: actual.append(g.bounds) self.assertEqual(actual, desired) # Test updating the coordinate system. update_crs_call_count = {WGS84: 1, Spherical: 0, None: 1} wrapped_state = [None, WrappedState.WRAPPED, WrappedState.UNWRAPPED, WrappedState.UNKNOWN] keywords = dict(archetype_crs=update_crs_call_count.keys(), wrapped_state=wrapped_state) for k in self.iter_product_keywords(keywords): dgvar = deepcopy(gvar) dgvar.update_crs = mock.Mock() dgvar.deepcopy = mock.Mock(return_value=dgvar) dgvar.copy = mock.Mock() archetype = mock.create_autospec(GeometryVariable, spec_set=True) archetype.wrapped_state = k.wrapped_state try: archetype.crs = k.archetype_crs() archetype.crs.wrap_or_unwrap = mock.Mock() except TypeError: archetype.crs = None _ = dgvar.prepare(archetype=archetype) desired_count = update_crs_call_count[k.archetype_crs] self.assertEqual(dgvar.update_crs.call_count, desired_count) if desired_count > 0: dgvar.update_crs.assert_called_once_with(archetype.crs) if k.archetype_crs is not None and k.wrapped_state not in (WrappedState.UNKNOWN, None): archetype.crs.wrap_or_unwrap.assert_called_once_with(archetype.wrapped_state, dgvar) else: if k.archetype_crs is not None: archetype.crs.wrap_or_unwrap.assert_not_called() dgvar.deepcopy.assert_called_once() dgvar.copy.assert_not_called() # Test identical object is returned if nothing happens. gvar = GeometryVariable() gvar.deepcopy = mock.Mock(spec=GeometryVariable.deepcopy) gvar.copy = mock.Mock(spec=GeometryVariable.copy) actual = gvar.prepare() self.assertNotEqual(id(actual), id(gvar)) gvar.deepcopy.assert_not_called() gvar.copy.assert_called_once() # Test exception for more than one geometry. gvar = mock.create_autospec(GeometryVariable, spec_set=True) gvar.size = 2 with self.assertRaises(RequestableFeature): GeometryVariable.prepare(gvar)
def test_get_intersection_state_boundaries(self): path_shp = self.path_state_boundaries geoms = [] with fiona.open(path_shp) as source: for record in source: geom = shape(record['geometry']) geoms.append(geom) gvar = GeometryVariable(value=geoms, dimensions='ngeom') gvar_sub = gvar.get_unioned() if gvar_sub is not None: subset = gvar_sub.get_value().flatten()[0] else: subset = None subset = MPI_COMM.bcast(subset) resolution = 2.0 keywords = dict(with_bounds=[False]) for k in self.iter_product_keywords(keywords): grid = self.get_gridxy_global(resolution=resolution, with_bounds=k.with_bounds) res = grid.get_intersection(subset) if not res.is_empty: self.assertTrue(res.get_mask().any()) else: self.assertIsInstance(res, GeometryVariable) if k.with_bounds: area = res.area if area is None: area = 0.0 else: area = area.sum() areas = MPI_COMM.gather(area) if MPI_RANK == 0: area_global = sum(areas) self.assertAlmostEqual(area_global, 1096.0819224080542) else: mask = res.get_mask() if mask is None: masked = 0 else: masked = mask.sum() masked = MPI_COMM.gather(masked) if MPI_RANK == 0: total_masked = sum(masked) self.assertEqual(total_masked, 858)
def test_crs(self): crs = WGS84() gvar = GeometryVariable(crs=crs, name='var') self.assertEqual(gvar.crs, crs) self.assertIn(crs.name, gvar.parent) gvar.crs = None self.assertIsNone(gvar.crs) self.assertEqual(len(gvar.parent), 1) self.assertNotIn(crs.name, gvar.parent) # Test coordinate system is maintained. gvar = GeometryVariable(crs=crs, name='var') vc = VariableCollection(variables=gvar) self.assertIn(crs.name, vc)
def run_do_remove_self_intersects(self, fixture, debug=False): poly = Polygon(fixture) if debug: gvar = GeometryVariable.from_shapely(poly) gvar.write_vector('/tmp/vector.shp') new_poly = do_remove_self_intersects_multi(poly) self.assertEqual( np.array(poly.exterior.coords).shape[0] - 1, np.array(new_poly.exterior.coords).shape[0]) self.assertTrue(new_poly.is_valid) if debug: GeometryVariable.from_shapely(new_poly).write_vector( '/tmp/new_vector.shp')
def test_write_variable_collection(self): # Attempt to write without a geometry variable. v = Variable('a', value=[1, 2], dimensions='bb') field = Field(variables=v) path = self.get_temporary_file_path('out.shp') with self.assertRaises(ValueError): field.write(path, driver=DriverVector) # Test writing a field with two-dimensional geometry storage. value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)] gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms') gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)]) var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a']) var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat']) var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon']) var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat']) field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data']) path = self.get_temporary_file_path('2d.shp') field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector) read = RequestDataset(uri=path).get() self.assertTrue(len(read) > 2) self.assertEqual(list(read.keys()), ['data', 'some_lats', 'some_lons', constants.VariableName.GEOMETRY_VARIABLE]) # Test writing a subset of the variables. path = self.get_temporary_file_path('limited.shp') value = [Point(1, 2), Point(3, 4), Point(5, 6)] gvar = GeometryVariable(value=value, name='points', dimensions='points') var1 = Variable('keep', value=[1, 2, 3], dimensions='points') var2 = Variable('remove', value=[4, 5, 6], dimensions='points') field = Field(variables=[var1, var2], geom=gvar, is_data=[var1]) field.write(path, variable_names=['keep'], driver=DriverVector) read = RequestDataset(uri=path).get() self.assertNotIn('remove', read) # Test using append. path = self.get_temporary_file_path('limited.shp') value = [Point(1, 2), Point(3, 4), Point(5, 6)] gvar = GeometryVariable(value=value, name='points', dimensions='points') var1 = Variable('keep', value=[1, 2, 3], dimensions='points') var2 = Variable('remove', value=[4, 5, 6], dimensions='points') field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2]) for idx in range(3): sub = field[{'points': idx}] if idx == 0: write_mode = MPIWriteMode.WRITE else: write_mode = MPIWriteMode.APPEND sub.write(path, write_mode=write_mode, driver=DriverVector) self.assertOGRFileLength(path, idx + 1)
def test_get_mask_from_intersects(self): poly = wkt.loads( 'POLYGON((-98.26574367088608142 40.19952531645570559,-98.71764240506330168 39.54825949367089066,-99.26257911392406186 39.16281645569620906,-99.43536392405064817 38.64446202531645724,-98.78409810126584034 38.33876582278481493,-98.23916139240508016 37.71408227848101546,-97.77397151898735217 37.67420886075949937,-97.62776898734178133 38.15268987341772799,-98.39865506329114453 38.52484177215190186,-98.23916139240508016 39.33560126582278826,-97.73409810126582897 39.58813291139241386,-97.52143987341773368 40.27927215189873777,-97.52143987341773368 40.27927215189873777,-98.26574367088608142 40.19952531645570559))') desired_mask = np.array([[True, True, False, True], [True, False, True, True], [True, True, False, True]]) dist = OcgDist() xdim = dist.create_dimension('x', 4, dist=True) ydim = dist.create_dimension('y', 3) dist.create_dimension('bounds', 2) dist.update_dimension_bounds() if MPI_RANK == 0: x = self.get_variable_x() y = self.get_variable_y() grid = Grid(x=x, y=y, abstraction='point', crs=WGS84()) pa = get_geometry_variable(grid) else: pa = None pa = variable_scatter(pa, dist) vm.create_subcomm_by_emptyable('test_get_mask_from_intersects', pa, is_current=True) if vm.is_null: self.assertTrue(pa.is_empty) return usi = [False] if env.USE_SPATIAL_INDEX: usi.append(True) keywords = dict(use_spatial_index=usi) for k in self.iter_product_keywords(keywords): ret = pa.get_mask_from_intersects(poly, use_spatial_index=k.use_spatial_index) desired_mask_local = desired_mask[slice(*ydim.bounds_local), slice(*xdim.bounds_local)] if MPI_RANK > 1: self.assertIsNone(ret) else: self.assertNumpyAll(desired_mask_local, ret) # This does not test a parallel operation. if MPI_RANK == 0: # Test pre-masked values in geometry are okay for intersects operation. value = [Point(1, 1), Point(2, 2), Point(3, 3)] value = np.ma.array(value, mask=[False, True, False], dtype=object) pa2 = GeometryVariable(value=value, dimensions='ngeom') b = box(0, 0, 5, 5) res = pa2.get_mask_from_intersects(b, use_spatial_index=k.use_spatial_index) self.assertNumpyAll(res, value.mask)
def test_create_ugid_global(self): ompi = OcgDist() m = ompi.create_dimension('m', 4) n = ompi.create_dimension('n', 70, dist=True) ompi.update_dimension_bounds() gvar = GeometryVariable(name='geom', dimensions=(m, n)) ugid = gvar.create_ugid_global('gid') if not gvar.is_empty: self.assertEqual(gvar.dist, ugid.dist) gathered = variable_gather(ugid) if MPI_RANK == 0: actual = gathered.get_value() self.assertEqual(actual.size, len(set(actual.flatten().tolist())))
def test_convert_to_self_intersecting(self): poly = Polygon(self.fixture_self_intersecting_polygon_coords) gvar = GeometryVariable.from_shapely(poly) without_si = gvar.convert_to(remove_self_intersects=True) gvar2 = without_si.convert_to() desired = do_remove_self_intersects_multi(poly) self.assertPolygonSimilar(gvar2.v()[0], desired)
def test_set_geom(self): f = Field() self.assertIsNone(f.crs) g = GeometryVariable(value=[Point(1, 2)], dimensions='geom', crs=Spherical()) f.set_geom(g)
def test_update_crs(self): # Test copying allows the CRS to be updated on the copy w/out changing the source CRS. desired = Spherical() gvar = GeometryVariable(value=[Point(1, 2)], name='geom', dimensions='geom') field = Field(crs=desired, geom=gvar) cfield = field.copy() self.assertEqual(cfield.crs, desired) new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326) cfield.update_crs(new_crs) self.assertEqual(field.crs, desired) # Test geometry crs update is called. mfield = Mock(Field) mfield.is_empty = False mfield.dimension_map = Mock(DimensionMap) mfield.grid = Mock(Grid) mfield.geom = Mock(GeometryVariable) mcrs = Mock(Spherical) Field.update_crs(mfield, mcrs) mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs) mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs) from_crs = Mock(WGS84) mfield.grid.update_crs.reset_mock() mfield.geom.update_crs.reset_mock() Field.update_crs(mfield, mcrs, from_crs=from_crs) mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=from_crs) mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
def test_system_with_time_data(self): """Test writing data with a time dimension.""" path = self.get_temporary_file_path('what.shp') t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time') geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='time') field = Field(variables=[t, geom], dimension_map={ 'time': { 'variable': 'time' }, 'geom': { 'variable': 'geom' } }) field.write(path, iter_kwargs={'variable': 'time'}, driver=DriverVector) rd = RequestDataset(uri=path) field2 = rd.get() # netcdftime worthlessness poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'], ['1-01-02 12:00:00', '1-01-03 12:00:00']] actual = field2['TIME'].get_value().tolist() res = [p == actual for p in poss] self.assertTrue(any(res))
def test_properties(self): # Test an empty properties dictionary is created if there are no data variables on the container/parent field. gvar = GeometryVariable(name='empty', value=[Point(1, 2)], ugid=20, dimensions='ngeom') info = Variable(name='info') gvar.parent.add_variable(info) sc = SpatialCollection() sc.add_field(Field(name='what'), gvar.parent) self.assertEqual(len(sc.properties), 1)
def geoms(self): geoms = GeometryVariable( name='geoms', value=[Point(100.972, 41.941), Point(102.898, 40.978)], dimensions='ngeom', crs=self.crs) return geoms
def test_convert_to_geometry_coordinates_multipolygon(self): p1 = 'Polygon ((-116.94238466549290933 52.12861711455555991, -82.00526805089285176 61.59075286434307372, -59.92695130138864101 31.0207758265680269, -107.72286778108455962 22.0438778075388484, -122.76523743459291893 37.08624746104720771, -116.94238466549290933 52.12861711455555991))' p2 = 'Polygon ((-63.08099655131782413 21.31602121140134898, -42.70101185946779765 9.42769680782217279, -65.99242293586783603 9.912934538580501, -63.08099655131782413 21.31602121140134898))' p1 = wkt.loads(p1) p2 = wkt.loads(p2) mp1 = MultiPolygon([p1, p2]) mp2 = mp1.buffer(0.1) geoms = [mp1, mp2] gvar = GeometryVariable(name='gc', value=geoms, dimensions='gd') # Test the element node connectivity arrays. results = [] for pack in [False, True]: gc = gvar.convert_to(pack=pack) self.assertEqual(gc.dimension_map.get_driver(), DriverKey.NETCDF_UGRID) self.assertTrue(gc.has_multi) self.assertIn(OcgisConvention.Name.MULTI_BREAK_VALUE, gc.cindex.attrs) # Test multi-break values are part of the element node connectivity arrays. actual = gc.cindex.get_value() for idx, ii in enumerate(actual.flat): self.assertGreater( np.sum(ii == OcgisConvention.Value.MULTI_BREAK_VALUE), 0) results.append(actual) self.assertIsNotNone(gc.x.get_value()) self.assertIsNotNone(gc.y.get_value()) maxes = [] for ii in actual.flat: maxes.append(ii.max()) actual_max = max(maxes) for c in [gc.x, gc.y]: self.assertEqual(c.size - 1, actual_max) geoms = list(gc.iter_geometries()) for ctr, g in enumerate(geoms): self.assertIsInstance(g[1], BaseMultipartGeometry) self.assertEqual(ctr, 1) self.assertPolygonSimilar(geoms[0][1], mp1) self.assertPolygonSimilar(geoms[1][1], mp2) for idx in range(len(results[0])): self.assertNumpyAll(results[0][idx], results[1][idx])
def test_unwrap(self): geom = box(195, -40, 225, -30) gvar = GeometryVariable(name='geoms', value=geom, crs=Spherical(), dimensions='geoms') gvar.wrap() self.assertEqual(gvar.get_value()[0].bounds, (-165.0, -40.0, -135.0, -30.0)) gvar.unwrap() self.assertEqual(gvar.get_value()[0].bounds, (195.0, -40.0, 225.0, -30.0))
def test_convert_to_geometry_coordinates_multipolygon_node_threshold(self): mp = wkt.loads(strings.S7) desired_count = len(get_split_polygon_by_node_threshold(mp, 10)) self.assertGreater(desired_count, len(mp)) gvar = GeometryVariable.from_shapely(mp) gc = gvar.convert_to(node_threshold=10) actual_count = gc.cindex.get_value()[0] actual_count = np.sum(actual_count == OcgisConvention.Value.MULTI_BREAK_VALUE) + 1 self.assertEqual(actual_count, desired_count)
def test_write_variable_collection_different_data_types(self): """Test multiple data types are handled by the shapefile write when melted is True.""" v_int = Variable(name='an_int', value=[1, 2, 3], dtype=int, dimensions='three') v_flt = Variable(name='a_float', value=[10., 20., 30.], dtype=float, dimensions='three') g = GeometryVariable(name='points', value=[Point(1, 2), Point(3, 4), Point(5, 6)], dimensions='three') field = Field(is_data=[v_int, v_flt], geom=g) self.assertEqual(len(field.data_variables), 2) path = self.get_temporary_file_path('foo.shp') field.write(path, driver='vector', iter_kwargs=dict(melted=True))
def test_convert_to_geometry_coordinates_multipolygon_node_threshold(self): mp = wkt.loads(strings.S7) desired_count = len(get_split_polygon_by_node_threshold(mp, 10)) self.assertGreater(desired_count, len(mp)) gvar = GeometryVariable.from_shapely(mp) gc = gvar.convert_to(node_threshold=10) actual_count = gc.cindex.get_value()[0] actual_count = np.sum( actual_count == OcgisConvention.Value.MULTI_BREAK_VALUE) + 1 self.assertEqual(actual_count, desired_count)
def test_geom_type_global(self): if MPI_SIZE != 3: raise SkipTest('MPI_SIZE != 3') geoms = [Point(1, 2), MultiPoint([Point(3, 4)]), Point(5, 6)] geom = GeometryVariable(name='geom', value=[geoms[MPI_RANK]], dimensions='geom') self.assertEqual(geom.geom_type, geoms[MPI_RANK].geom_type) self.assertEqual(geom.geom_type_global, geoms[1].geom_type)
def test_convert_to_geometry_coordinates_multipolygon(self): p1 = 'Polygon ((-116.94238466549290933 52.12861711455555991, -82.00526805089285176 61.59075286434307372, -59.92695130138864101 31.0207758265680269, -107.72286778108455962 22.0438778075388484, -122.76523743459291893 37.08624746104720771, -116.94238466549290933 52.12861711455555991))' p2 = 'Polygon ((-63.08099655131782413 21.31602121140134898, -42.70101185946779765 9.42769680782217279, -65.99242293586783603 9.912934538580501, -63.08099655131782413 21.31602121140134898))' p1 = wkt.loads(p1) p2 = wkt.loads(p2) mp1 = MultiPolygon([p1, p2]) mp2 = mp1.buffer(0.1) geoms = [mp1, mp2] gvar = GeometryVariable(name='gc', value=geoms, dimensions='gd') # Test the element node connectivity arrays. results = [] for pack in [False, True]: gc = gvar.convert_to(pack=pack) self.assertEqual(gc.dimension_map.get_driver(), DriverKey.NETCDF_UGRID) self.assertTrue(gc.has_multi) self.assertIn(OcgisConvention.Name.MULTI_BREAK_VALUE, gc.cindex.attrs) # Test multi-break values are part of the element node connectivity arrays. actual = gc.cindex.get_value() for idx, ii in enumerate(actual.flat): self.assertGreater(np.sum(ii == OcgisConvention.Value.MULTI_BREAK_VALUE), 0) results.append(actual) self.assertIsNotNone(gc.x.get_value()) self.assertIsNotNone(gc.y.get_value()) maxes = [] for ii in actual.flat: maxes.append(ii.max()) actual_max = max(maxes) for c in [gc.x, gc.y]: self.assertEqual(c.size - 1, actual_max) geoms = list(gc.iter_geometries()) for ctr, g in enumerate(geoms): self.assertIsInstance(g[1], BaseMultipartGeometry) self.assertEqual(ctr, 1) self.assertPolygonSimilar(geoms[0][1], mp1) self.assertPolygonSimilar(geoms[1][1], mp2) for idx in range(len(results[0])): self.assertNumpyAll(results[0][idx], results[1][idx])
def test_get_unioned_spatial_average_parallel(self): if MPI_SIZE != 8: raise SkipTest('MPI_SIZE != 8') dist = OcgDist() geom_count = dist.create_dimension('geom_count', size=8, dist=True) time_count = dist.create_dimension('time', size=3) dist.update_dimension_bounds() if not geom_count.is_empty: gvar = GeometryVariable( value=[Point(1.0, 1.0).buffer(MPI_RANK + 2)] * len(geom_count), dimensions=geom_count) value = np.zeros((len(time_count), len(geom_count)), dtype=float) for ii in range(value.shape[0]): value[ii] = [MPI_RANK + 1 + ii + 1] * len(geom_count) data = Variable(name='data', value=value, dtype=float, dimensions=[time_count, geom_count]) else: gvar = GeometryVariable(dimensions=geom_count) data = Variable(name='data', dimensions=[time_count, geom_count], dtype=float) gvar.parent.add_variable(data) self.assertTrue(gvar.is_empty == data.is_empty == gvar.parent.is_empty) with vm.scoped_by_emptyable('union', gvar): if vm.is_null: unioned = None else: unioned = gvar.get_unioned(spatial_average='data') if unioned is not None: self.assertIsInstance(unioned, GeometryVariable) actual = unioned.parent[data.name] self.assertAlmostEqual(actual.get_value().max(), 5.5466666666666677) else: self.assertIsNone(unioned)
def get_variable_collection(self, **kwargs): parent = VariableCollection(**kwargs) for n, v in list(self.metadata_source['variables'].items()): SourcedVariable(name=n, request_dataset=self.rd, parent=parent) GeometryVariable(name=DimensionName.GEOMETRY_DIMENSION, request_dataset=self.rd, parent=parent) crs = self.get_crs(self.metadata_source) if crs is not None: parent.add_variable(crs) return parent
def test_get_spatial_subset_circular_geometries(self): """Test circular geometries. They were causing wrapping errors.""" geoms = TestGeom.get_geometry_dictionaries() rd = self.test_data.get_rd('cancm4_tas') buffered = [element['geom'].buffer(rd.get().grid.resolution * 2) for element in geoms] for buff in buffered: ss = SpatialSubsetOperation(rd.get(), wrap=False) gvar = GeometryVariable(value=buff, name='geom', dimensions='dim', crs=WGS84()) ret = ss.get_spatial_subset('intersects', gvar) self.assertTrue(np.all(ret.grid.x.get_value() >= 0))
def test_system_conform_units(self): """Test conforming units on data read from shapefile.""" path = self.get_temporary_file_path('temps.shp') gvar = GeometryVariable(value=[Point(1, 2), Point(3, 4)], dimensions='g', name='geom') var = Variable(name='temp', value=[10., 20.], dimensions='g') field = Field(variables=[gvar, var], geom=gvar, is_data=var) field.write(path, driver=DriverVector) field = RequestDataset(path, units='celsius', variable='temp', conform_units_to='fahrenheit').get() self.assertNumpyAllClose(field['temp'].get_value(), np.array([50., 68.]))
def test_convert_to_geometry_coordinates_points(self): pt1 = Point(1, 2, 3) pt2 = Point(3, 4, 4) pt3 = Point(5, 6, 5) pt4 = Point(7, 8, 6) crs = WGS84() value = [pt1, pt2, pt3, pt4] mask = [True, False, True, False] gvar = GeometryVariable(value=value, dimensions='ngeom', crs=crs, mask=mask) gvar_mask = gvar.get_mask().tolist() self.assertEqual(mask, gvar_mask) actual = gvar.convert_to() self.assertEqual(actual.get_mask().tolist(), gvar_mask) for actual_geom, desired_geom in zip(actual.get_geometry_iterable(use_mask=False), value): self.assertEqual(actual_geom[1], desired_geom) self.assertEqual(actual.crs, crs)
def get_geometryvariable_with_parent(): vpa = np.array([None, None, None]) vpa[:] = [Point(1, 2), Point(3, 4), Point(5, 6)] value = np.arange(0, 30).reshape(10, 3) tas = Variable(name='tas', value=value, dimensions=['time', 'ngeom']) backref = Field(variables=[tas]) pa = GeometryVariable(value=vpa, parent=backref, name='point', dimensions='ngeom') backref[pa.name] = pa return pa
def test_write_variable_collection(self): # Attempt to write without a geometry variable. v = Variable('a', value=[1, 2], dimensions='bb') field = Field(variables=v) path = self.get_temporary_file_path('out.shp') with self.assertRaises(ValueError): field.write(path, driver=DriverVector) # Test writing a field with two-dimensional geometry storage. value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)] gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms') gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)]) var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a']) var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat']) var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon']) var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat']) field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data']) path = self.get_temporary_file_path('2d.shp') field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector) read = RequestDataset(uri=path).get() self.assertTrue(len(read) > 2) self.assertEqual(list(read.keys()), ['data', 'some_lats', 'some_lons', constants.DimensionName.GEOMETRY_DIMENSION]) # Test writing a subset of the variables. path = self.get_temporary_file_path('limited.shp') value = [Point(1, 2), Point(3, 4), Point(5, 6)] gvar = GeometryVariable(value=value, name='points', dimensions='points') var1 = Variable('keep', value=[1, 2, 3], dimensions='points') var2 = Variable('remove', value=[4, 5, 6], dimensions='points') field = Field(variables=[var1, var2], geom=gvar, is_data=[var1]) field.write(path, variable_names=['keep'], driver=DriverVector) read = RequestDataset(uri=path).get() self.assertNotIn('remove', read) # Test using append. path = self.get_temporary_file_path('limited.shp') value = [Point(1, 2), Point(3, 4), Point(5, 6)] gvar = GeometryVariable(value=value, name='points', dimensions='points') var1 = Variable('keep', value=[1, 2, 3], dimensions='points') var2 = Variable('remove', value=[4, 5, 6], dimensions='points') field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2]) for idx in range(3): sub = field[{'points': idx}] if idx == 0: write_mode = MPIWriteMode.WRITE else: write_mode = MPIWriteMode.APPEND sub.write(path, write_mode=write_mode, driver=DriverVector) self.assertOGRFileLength(path, idx + 1)
def test_update_crs(self): # Test copying allows the CRS to be updated on the copy w/out changing the source CRS. desired = Spherical() gvar = GeometryVariable(value=[Point(1, 2)], name='geom', dimensions='geom') field = Field(crs=desired, geom=gvar) cfield = field.copy() self.assertEqual(cfield.crs, desired) new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326) cfield.update_crs(new_crs) self.assertEqual(field.crs, desired)
def test_convert_to_geometry_coordinates_polygon_interior(self): ph = self.fixture_polygon_with_hole gvar = GeometryVariable.from_shapely(ph) desired_count = len(GeometrySplitter(ph).split()) keywords = dict(split_interiors=[True, False]) for k in self.iter_product_keywords(keywords): try: gc = gvar.convert_to(split_interiors=k.split_interiors) except ValueError: self.assertFalse(k.split_interiors) continue actual_count = gc.cindex.get_value()[0] actual_count = np.sum(actual_count == OcgisConvention.Value.MULTI_BREAK_VALUE) + 1 self.assertEqual(actual_count, desired_count)
def test_init(self): # Test empty. gvar = GeometryVariable() self.assertEqual(gvar.dtype, object) gvar = self.get_geometryvariable() self.assertIsInstance(gvar.get_masked_value(), MaskedArray) self.assertEqual(gvar.ndim, 1) # The geometry variable should set itself as the representative geometry on its parent field if that parent does # not have a representative geometry set. self.assertIsNotNone(gvar.parent.geom) # Test with a parent that already has a geometry. field = Field() field.set_geom(GeometryVariable(name='empty')) gvar = self.get_geometryvariable(parent=field) self.assertEqual(field.geom.name, 'empty') self.assertIn(gvar.name, field) # Test passing a "crs". gvar = self.get_geometryvariable(crs=WGS84(), name='my_geom', dimensions='ngeom') self.assertEqual(gvar.crs, WGS84()) # Test using lines. line1 = LineString([(0, 0), (1, 1)]) line2 = LineString([(1, 1), (2, 2)]) gvar = GeometryVariable(value=[line1, line2], dimensions='two') self.assertTrue(gvar.get_value()[1].almost_equals(line2)) self.assertEqual(gvar.geom_type, line1.geom_type) lines = MultiLineString([line1, line2]) lines2 = [lines, lines] for actual in [lines, lines2, lines]: gvar2 = GeometryVariable(value=actual, dimensions='ngeom') self.assertTrue(gvar2.get_value()[0].almost_equals(lines)) self.assertEqual(gvar2.geom_type, lines.geom_type) self.assertTrue(gvar2.shape[0] > 0) self.assertIsNone(gvar2.get_mask())
def get_spatial_subset(self, operation, geom, use_spatial_index=env.USE_SPATIAL_INDEX, buffer_value=None, buffer_crs=None, geom_crs=None, select_nearest=False, optimized_bbox_subset=False): """ Perform a spatial subset operation on ``target``. :param str operation: Either ``'intersects'`` or ``'clip'``. :param geom: The input geometry object to use for subsetting of ``target``. :type geom: :class:`shapely.geometry.base.BaseGeometry` | :class:`ocgis.GeometryVariable` :param bool use_spatial_index: If ``True``, use an ``rtree`` spatial index. :param bool select_nearest: If ``True``, select the geometry nearest ``polygon`` using :meth:`shapely.geometry.base.BaseGeometry.distance`. :rtype: Same as ``target``. If ``target`` is a :class:`ocgis.RequestDataset`, then a :class:`ocgis.interface.base.field.Field` will be returned. :param float buffer_value: The buffer radius to use in units of the coordinate system of ``subset_sdim``. :param buffer_crs: If provided, then ``buffer_value`` are not in units of the coordinate system of ``subset_sdim`` but in units of ``buffer_crs``. :param geom_crs: The coordinate reference system for the subset geometry. :type geom_crs: :class:`ocgis.crs.CRS` :param bool select_nearest: If ``True``, following the spatial subset operation, select the nearest geometry in the subset data to ``geom``. Centroid-based distance is used. :type buffer_crs: :class:`ocgis.interface.base.crs.CoordinateReferenceSystem` :param bool optimized_bbox_subset: If ``True``, only do a bounding box subset and do not perform more complext GIS subset operations such as constructing a spatial index. :raises: ValueError """ if not isinstance(geom, GeometryVariable): geom = GeometryVariable(value=geom, name='geom', dimensions='one', crs=geom_crs) if geom.get_value().flatten().shape != (1,): msg = 'Only one subset geometry allowed. The shape of the geometry variable is {}.'.format(geom.shape) raise ValueError(msg) if optimized_bbox_subset: if self.field.grid is None: msg = 'Subset operation must be performed on a grid when "optimized_bbox_subset=True".' raise ValueError(msg) if operation != 'intersects': msg = 'Only "intersects" spatial operations when "optimized_bbox_subset=True".' raise ValueError(msg) # Buffer the subset if a buffer value is provided. if buffer_value is not None: geom = self._get_buffered_geometry_(geom, buffer_value, buffer_crs=buffer_crs) prepared = self._prepare_geometry_(geom) base_geometry = prepared.get_value().flatten()[0] # Prepare the target field. self._prepare_target_() # execute the spatial operation if operation == 'intersects': if self.field.grid is None: ret = self.field.geom.get_intersects(base_geometry, use_spatial_index=use_spatial_index, cascade=True).parent else: ret = self.field.grid.get_intersects(base_geometry, cascade=True, optimized_bbox_subset=optimized_bbox_subset).parent elif operation in ('clip', 'intersection'): if self.field.grid is None: ret = self.field.geom.get_intersection(base_geometry, use_spatial_index=use_spatial_index, cascade=True).parent else: ret = self.field.grid.get_intersection(base_geometry, cascade=True) # An intersection with a grid returns a geometry variable. Set this on the field. ret.parent.set_geom(ret) ret = ret.parent else: msg = 'The spatial operation "{0}" is not supported.'.format(operation) raise ValueError(msg) with vm.scoped_by_emptyable('return finalize', ret): if not vm.is_null: # Select the nearest geometry if requested. if select_nearest: ret.set_abstraction_geom() ret = ret.geom.get_nearest(base_geometry).parent # check for rotated pole and convert back to default CRS if self._original_rotated_pole_state is not None and self.output_crs == 'input': ret.update_crs(self._original_rotated_pole_state) # wrap the data... if self._get_should_wrap_(ret): ret.wrap() # convert the coordinate system if requested... if self.should_update_crs: ret.update_crs(self.output_crs) return ret
def from_records(cls, records, schema=None, crs=UNINITIALIZED, uid=None, union=False, data_model=None): """ Create a :class:`~ocgis.Field` from Fiona-like records. :param records: A sequence of records returned from an Fiona file object. :type records: `sequence` of :class:`dict` :param schema: A Fiona-like schema dictionary. If ``None`` and any records properties are ``None``, then this must be provided. :type schema: dict >>> schema = {'geometry': 'Point', 'properties': {'UGID': 'int', 'NAME', 'str:4'}} :param crs: If :attr:`ocgis.constants.UNINITIALIZED`, default to :attr:`ocgis.env.DEFAULT_COORDSYS`. :type crs: :class:`dict` | :class:`~ocgis.variable.crs.AbstractCoordinateReferenceSystem` :param str uid: If provided, use this attribute name as the unique identifier. Otherwise search for :attr:`env.DEFAULT_GEOM_UID` and, if not present, construct a 1-based identifier with this name. :param bool union: If ``True``, union the geometries from records yielding a single geometry with a unique identifier value of ``1``. :param str data_model: See :meth:`~ocgis.driver.nc.create_typed_variable_from_data_model`. :returns: Field object constructed from records. :rtype: :class:`~ocgis.Field` """ if uid is None: uid = env.DEFAULT_GEOM_UID if isinstance(crs, dict): crs = CoordinateReferenceSystem(value=crs) elif crs == UNINITIALIZED: crs = env.DEFAULT_COORDSYS if union: deque_geoms = None deque_uid = [1] else: # Holds geometry objects. deque_geoms = deque() # Holds unique identifiers. deque_uid = deque() build = True for ctr, record in enumerate(records, start=1): # Get the geometry from a keyword present on the input dictionary or construct from the coordinates # sequence. try: current_geom = record['geom'] except KeyError: current_geom = shape(record['geometry']) if union: if build: deque_geoms = current_geom else: deque_geoms = deque_geoms.union(current_geom) else: deque_geoms.append(current_geom) # Set up the properties array if build: build = False if uid in record['properties']: has_uid = True else: has_uid = False # The geometry unique identifier may be present as a property. Otherwise the enumeration counter is used for # the identifier. if not union: if has_uid: to_append = int(record['properties'][uid]) else: to_append = ctr deque_uid.append(to_append) # If we are unioning, the target geometry is not yet a sequence. if union: deque_geoms = [deque_geoms] # Dimension for the outgoing field. if union: size = 1 else: size = ctr dim = Dimension(name=DimensionName.GEOMETRY_DIMENSION, size=size) # Set default geometry type if no schema is provided. if schema is None: geom_type = 'auto' else: geom_type = schema['geometry'] geom = GeometryVariable(value=deque_geoms, geom_type=geom_type, dimensions=dim) uid = create_typed_variable_from_data_model('int', data_model=data_model, name=uid, value=deque_uid, dimensions=dim) geom.set_ugid(uid) field = Field(geom=geom, crs=crs) # All records from a unioned geometry are not relevant. if not union: from ocgis.driver.vector import get_dtype_from_fiona_type, get_fiona_type_from_pydata if schema is None: has_schema = False else: has_schema = True for idx, record in enumerate(records): if idx == 0 and not has_schema: schema = {'properties': OrderedDict()} for k, v in list(record['properties'].items()): schema['properties'][k] = get_fiona_type_from_pydata(v) if idx == 0: for k, v in list(schema['properties'].items()): if k == uid.name: continue dtype = get_dtype_from_fiona_type(v, data_model=data_model) var = Variable(name=k, dtype=dtype, dimensions=dim) if v.startswith('str:'): var.set_string_max_length_global(value=int(v.split(':')[1])) field.add_variable(var) for k, v in list(record['properties'].items()): if k == uid.name: continue if v is None: # Mask the value if it is None. NULLs are allowed in OGR Vector files, but they do not translate # well to Python. Strings are generally okay but floats/ints case problems. field[k].get_mask(create=True)[idx] = v else: # Set the associated field value. field[k].get_value()[idx] = v data_variables = [uid.name] if not union: data_variables += [k for k in list(schema['properties'].keys()) if k != uid.name] field.append_to_tags(TagName.DATA_VARIABLES, data_variables, create=True) return field
def iter_src_grid_subsets(self, yield_dst=False, yield_idx=None): """ Yield source grid subset using the extent of its associated destination grid subset. :param bool yield_dst: If ``True``, yield the destination subset as well as the source grid subset. :param int yield_idx: If a zero-based integer, only yield for this chunk index and skip everything else. :rtype: tuple(:class:`ocgis.spatial.grid.AbstractGrid`, `slice-like`) """ if yield_dst: yield_slice = True else: yield_slice = False buffer_value = self.buffer_value dst_grid_wrapped_state = self.dst_grid.wrapped_state dst_grid_crs = self.dst_grid.crs # Use a destination grid iterator if provided. if self.iter_dst is not None: iter_dst = self.iter_dst(self, yield_slice=yield_slice, yield_idx=yield_idx) else: iter_dst = self.iter_dst_grid_subsets(yield_slice=yield_slice, yield_idx=yield_idx) # Loop over each destination grid subset. ocgis_lh(logger='grid_chunker', msg='starting "for yld in iter_dst"', level=logging.DEBUG) for iter_dst_ctr, yld in enumerate(iter_dst, start=1): ocgis_lh(msg=["iter_dst_ctr", iter_dst_ctr], level=logging.DEBUG) if yield_slice: dst_grid_subset, dst_slice = yld else: dst_grid_subset = yld # All masked destinations are very problematic for ESMF with vm.scoped_by_emptyable('global mask', dst_grid_subset): if not vm.is_null: if dst_grid_subset.has_mask_global: if dst_grid_subset.has_mask and dst_grid_subset.has_masked_values: all_masked = dst_grid_subset.get_mask().all() else: all_masked = False all_masked_gather = vm.gather(all_masked) if vm.rank == 0: if all(all_masked_gather): exc = ValueError("Destination subset all masked") try: raise exc finally: vm.abort(exc=exc) dst_box = None with vm.scoped_by_emptyable('extent_global', dst_grid_subset): if not vm.is_null: # Use the extent of the polygon for determining the bounding box. This ensures conservative # regridding will be fully mapped. if isinstance(dst_grid_subset, AbstractGeometryCoordinates): target_grid = dst_grid_subset.parent.grid else: target_grid = dst_grid_subset # Try to reduce the coordinates in the case of unstructured grid data. if hasattr(target_grid, 'reduce_global') and Topology.POLYGON in target_grid.abstractions_available: ocgis_lh(logger='grid_chunker', msg='starting reduce_global for dst_grid_subset', level=logging.DEBUG) target_grid = target_grid.reduce_global() ocgis_lh(logger='grid_chunker', msg='finished reduce_global for dst_grid_subset', level=logging.DEBUG) extent_global = target_grid.parent.attrs.get('extent_global') if extent_global is None: with grid_abstraction_scope(target_grid, Topology.POLYGON): extent_global = target_grid.extent_global if self.check_contains: dst_box = box(*target_grid.extent_global) sub_box = box(*extent_global) if buffer_value is not None: # Use the envelope! A buffer returns "fancy" borders. We just want to expand the bounding box. sub_box = sub_box.buffer(buffer_value).envelope ocgis_lh(msg=str(sub_box.bounds), level=logging.DEBUG, logger='grid_chunker') else: sub_box, dst_box = [None, None] live_ranks = vm.get_live_ranks_from_object(dst_grid_subset) sub_box = vm.bcast(sub_box, root=live_ranks[0]) if self.check_contains: dst_box = vm.bcast(dst_box, root=live_ranks[0]) sub_box = GeometryVariable.from_shapely(sub_box, is_bbox=True, wrapped_state=dst_grid_wrapped_state, crs=dst_grid_crs) ocgis_lh(logger='grid_chunker', msg='starting "self.src_grid.get_intersects"', level=logging.DEBUG) src_grid_subset, src_grid_slice = self.src_grid.get_intersects(sub_box, keep_touches=False, cascade=False, optimized_bbox_subset=self.optimized_bbox_subset, return_slice=True) ocgis_lh(logger='grid_chunker', msg='finished "self.src_grid.get_intersects"', level=logging.DEBUG) # Reload the data using a new source index distribution. if hasattr(src_grid_subset, 'reduce_global') and src_grid_subset.cindex is not None: # Only redistribute if we have one live rank. if self.redistribute and len(vm.get_live_ranks_from_object(src_grid_subset)) > 0: ocgis_lh(logger='grid_chunker', msg='starting redistribute', level=logging.DEBUG) topology = src_grid_subset.abstractions_available[Topology.POLYGON] cindex = topology.cindex redist_dimname = self.src_grid.abstractions_available[Topology.POLYGON].element_dim.name if src_grid_subset.is_empty: redist_dim = None else: redist_dim = topology.element_dim redistribute_by_src_idx(cindex, redist_dimname, redist_dim) ocgis_lh(logger='grid_chunker', msg='finished redistribute', level=logging.DEBUG) with vm.scoped_by_emptyable('src_grid_subset', src_grid_subset): if not vm.is_null: if not self.allow_masked: gmask = src_grid_subset.get_mask() if gmask is not None and gmask.any(): raise ValueError('Masked values in source grid subset.') if self.check_contains: src_box = box(*src_grid_subset.extent_global) if not does_contain(src_box, dst_box): raise ValueError('Contains check failed.') # Try to reduce the coordinates in the case of unstructured grid data. if hasattr(src_grid_subset, 'reduce_global') and src_grid_subset.cindex is not None: ocgis_lh(logger='grid_chunker', msg='starting reduce_global', level=logging.DEBUG) src_grid_subset = src_grid_subset.reduce_global() ocgis_lh(logger='grid_chunker', msg='finished reduce_global', level=logging.DEBUG) else: pass # src_grid_subset = VariableCollection(is_empty=True) if src_grid_subset.is_empty: src_grid_slice = None else: src_grid_slice = {src_grid_subset.dimensions[ii].name: src_grid_slice[ii] for ii in range(src_grid_subset.ndim)} if yield_dst: yld = (src_grid_subset, src_grid_slice, dst_grid_subset, dst_slice) else: yld = src_grid_subset, src_grid_slice yield yld