def test_spatial(self): ## intersects geom = make_poly((37.5,39.5),(-104.5,-102.5)) ret = self.get_ret(kwds={'geom':geom}) ref = ret[1] gids = set([6,7,10,11]) ret_gids = set(ref.variables[self.var].spatial.vector.uid.compressed()) intersection = gids.intersection(ret_gids) self.assertEqual(len(intersection),4) self.assertTrue(np.all(ref.variables[self.var].value[0,0,:,:] == np.array([[1.0,2.0],[3.0,4.0]]))) ## intersection geom = make_poly((38,39),(-104,-103)) ret = self.get_ret(kwds={'geom':geom,'spatial_operation':'clip'}) self.assertEqual(len(ret[1].variables[self.var].spatial.vector.uid.compressed()),4) self.assertEqual(ret[1].variables[self.var].value.shape,(61,2,2,2)) ref = ret[1].variables[self.var].value self.assertTrue(np.all(ref[0,0,:,:] == np.array([[1,2],[3,4]],dtype=float))) ## compare areas to intersects returns ref = ret[1].variables[self.var] intersection_areas = [g.area for g in ref.spatial.vector.geom.flat] for ii in intersection_areas: self.assertAlmostEqual(ii,0.25) ## intersection + aggregation geom = make_poly((38,39),(-104,-103)) ret = self.get_ret(kwds={'geom':geom,'spatial_operation':'clip','aggregate':True}) ref = ret[1] self.assertEqual(len(ref.variables[self.var].spatial.vector.uid.flatten()),1) self.assertEqual(ref.variables[self.var].spatial.vector.geom.flatten()[0].area,1.0) self.assertEqual(ref.variables[self.var].value.flatten().mean(),2.5)
def test_regrid_field_different_grid_shapes(self): """Test regridding a downscaled dataset to GCM output. The input and output grids have different shapes.""" downscaled = self.test_data.get_rd('maurer_2010_tas') downscaled.time_region = {'month': [2], 'year': [1990]} downscaled = downscaled.get() poly = make_poly([37, 43], [-104, -94]) downscaled = downscaled.grid.get_intersects(poly).parent downscaled.unwrap() downscaled.set_crs(Spherical()) gcm = self.test_data.get_rd('cancm4_tas') gcm = gcm.get() poly = make_poly([37, 43], [-104 + 360, -94 + 360]) gcm = gcm.grid.get_intersects(poly).parent gcm.set_crs(Spherical()) self.assertIsNone(downscaled.grid.get_mask()) self.assertIsNone(gcm.grid.get_mask()) from ocgis.regrid.base import regrid_field regridded = regrid_field(downscaled, gcm) dv = regridded.data_variables[0] self.assertEqual(dv.shape, (28, 3, 5)) self.assertEqual(dv.name, 'tas') vmask = dv.get_mask() self.assertEqual(vmask.sum(), 252)
def test_get_buffered_subset_sdim(self): proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs' buffer_crs_list = [None, CoordinateReferenceSystem(proj4=proj4)] poly = make_poly((36, 44), (-104, -95)) for buffer_crs in buffer_crs_list: subset_sdim = SpatialDimension.from_records([{'geom': poly, 'properties': {'UGID': 1}}], crs=CFWGS84()) self.assertEqual(subset_sdim.crs, CFWGS84()) if buffer_crs is None: buffer_value = 1 else: buffer_value = 10 ret = SpatialSubsetOperation._get_buffered_subset_sdim_(subset_sdim, buffer_value, buffer_crs=buffer_crs) ref = ret.geom.polygon.value[0, 0] if buffer_crs is None: self.assertEqual(ref.bounds, (-105.0, 35.0, -94.0, 45.0)) else: self.assertNumpyAllClose(np.array(ref.bounds), np.array((-104.00013263459613, 35.9999147913708, -94.99986736540386, 44.00008450528758))) self.assertEqual(subset_sdim.crs, ret.crs) # check deepcopy ret.geom.polygon.value[0, 0] = make_poly((1, 2), (3, 4)) ref_buffered = ret.geom.polygon.value[0, 0] ref_original = subset_sdim.geom.polygon.value[0, 0] with self.assertRaises(AssertionError): self.assertNumpyAllClose(np.array(ref_buffered.bounds), np.array(ref_original.bounds))
def test_get_buffered_geometry(self): proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs' buffer_crs_list = [None, CoordinateReferenceSystem(proj4=proj4)] poly = make_poly((36, 44), (-104, -95)) for buffer_crs in buffer_crs_list: gvar = GeometryVariable(value=poly, name='geoms', dimensions='dim', crs=WGS84()) self.assertEqual(gvar.crs, WGS84()) if buffer_crs is None: buffer_value = 1 else: buffer_value = 10 ret = SpatialSubsetOperation._get_buffered_geometry_(gvar, buffer_value, buffer_crs=buffer_crs) ref = ret.get_value()[0] if buffer_crs is None: self.assertEqual(ref.bounds, (-105.0, 35.0, -94.0, 45.0)) else: self.assertNumpyAllClose(np.array(ref.bounds), np.array( (-104.00013263459613, 35.9999147913708, -94.99986736540386, 44.00008450528758))) self.assertEqual(gvar.crs, ret.crs) # check deepcopy ret.get_value()[0] = make_poly((1, 2), (3, 4)) ref_buffered = ret.get_value()[0] ref_original = gvar.get_value()[0] with self.assertRaises(AssertionError): self.assertNumpyAllClose(np.array(ref_buffered.bounds), np.array(ref_original.bounds))
def __init__(self, *args, **kwargs): super(GeometryWrapper, self).__init__(*args, **kwargs) self.right_clip = make_poly((-90, 90), (self.wrap_axis, 360)) self.left_clip = make_poly((-90, 90), (-self.wrap_axis, self.wrap_axis)) self.clip1 = make_poly((-90, 90), (-self.wrap_axis, self.center_axis)) self.clip2 = make_poly((-90, 90), (self.center_axis, self.wrap_axis))
def unwrap_geoms(geoms,left_max_x_bound): clip1 = make_poly((-90,90),(-180,left_max_x_bound)) clip2 = make_poly((-90,90),(left_max_x_bound,180)) def _shift_(polygon): coords = np.array(polygon.exterior.coords) coords[:,0] = coords[:,0] + 360 return(Polygon(coords)) def _transform_(geom,lon_cutoff): ## return the geometry iterator it = _get_iter_(geom) ## loop through the polygons determining if any coordinates need to be ## shifted and flag accordingly. adjust = False for polygon in it: coords = np.array(polygon.exterior.coords) if np.any(coords[:,0] < lon_cutoff): adjust = True break ## wrap the polygon if requested if adjust: ## intersection with the two regions left = geom.intersection(clip1) right = geom.intersection(clip2) ## pull out the right side polygons right_polygons = [poly for poly in _get_iter_(right)] ## adjust polygons falling the left window if isinstance(left,Polygon): left_polygons = [_shift_(left)] else: left_polygons = [] for polygon in left: left_polygons.append(_shift_(polygon)) ## merge polygons into single unit try: ret = MultiPolygon(left_polygons + right_polygons) except TypeError: left = filter(lambda x: type(x) != LineString,left_polygons) right = filter(lambda x: type(x) != LineString,right_polygons) ret = MultiPolygon(left+right) ## if polygon does not need adjustment, just return it. else: ret = geom return(ret) ## update the polygons in place for geom in geoms: geom['geom'] = _transform_(geom['geom'],left_max_x_bound)
def test_get_wrapped_state_from_geometry(self): geoms = [Point(-130, 40), MultiPoint([Point(-130, 40), Point(30, 50)]), make_poly((30, 40), (-130, -120)), MultiPolygon([make_poly((30, 40), (-130, -120)), make_poly((30, 40), (130, 160))])] for geom in geoms: ret = WrappableCoordinateReferenceSystem._get_wrapped_state_from_geometry_(geom) self.assertEqual(ret, WrappableCoordinateReferenceSystem._flag_wrapped) pt = Point(270, 50) ret = WrappableCoordinateReferenceSystem._get_wrapped_state_from_geometry_(pt) self.assertEqual(ret, WrappableCoordinateReferenceSystem._flag_unwrapped)
def test_get_wrapped_state_from_geometry(self): geoms = [Point(-130, 40), MultiPoint([Point(-130, 40), Point(30, 50)]), make_poly((30, 40), (-130, -120)), MultiPolygon([make_poly((30, 40), (-130, -120)), make_poly((30, 40), (130, 160))])] c = CoordinateReferenceSystem for geom in geoms: ret = c._get_wrapped_state_from_geometry_(geom) self.assertEqual(ret, WrappedState.WRAPPED) pt = Point(270, 50) ret = c._get_wrapped_state_from_geometry_(pt) self.assertEqual(ret, WrappedState.UNWRAPPED)
def test_geom(self): geom = make_poly((37.762, 38.222), (-102.281, -101.754)) g = Geom(geom) self.assertEqual(type(g.value), GeometryDataset) g.value = None self.assertEqual(None, g.value) g = Geom(None) self.assertEqual(g.value, None) self.assertEqual(str(g), 'geom=None') g = Geom('-120|40|-110|50') self.assertEqual(g.value.spatial.geom[0].bounds, (-120.0, 40.0, -110.0, 50.0)) self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0') g = Geom('mi_watersheds') self.assertEqual(str(g), 'geom=mi_watersheds') geoms = ShpCabinet().get_geoms('mi_watersheds') g = Geom('mi_watersheds') self.assertEqual(len(g.value), len(geoms)) su = SelectUgid([1, 2, 3]) g = Geom('mi_watersheds', select_ugid=su) self.assertEqual(len(g.value), 3) geoms = GeometryDataset(uid=[1, 2], geom=[geom, geom]) g = Geom(geoms) bbox = [-120, 40, -110, 50] g = Geom(bbox) self.assertEqual(g.value.spatial.geom[0].bounds, tuple(map(float, bbox)))
def test_init(self): geom = make_poly((37.762, 38.222), (-102.281, -101.754)) g = Geom(geom) self.assertEqual(type(g.value), tuple) self.assertIsInstance(g.value[0], SpatialDimension) g.value = None self.assertEqual(None, g.value) g = Geom(None) self.assertEqual(g.value, None) self.assertEqual(str(g), 'geom=None') g = Geom('-120|40|-110|50') self.assertEqual(g.value[0].geom.polygon.value[0, 0].bounds, (-120.0, 40.0, -110.0, 50.0)) self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0') g = Geom('state_boundaries') self.assertEqual(str(g), 'geom="state_boundaries"') geoms = list(ShpCabinetIterator('state_boundaries')) g = Geom('state_boundaries') self.assertEqual(len(list(g.value)), len(geoms)) sci = ShpCabinetIterator(key='state_boundaries') self.assertFalse(sci.as_spatial_dimension) g = Geom(sci) for _ in range(2): for ii, element in enumerate(g.value): self.assertIsInstance(element, SpatialDimension) self.assertGreater(ii, 10) su = GeomSelectUid([1, 2, 3]) g = Geom('state_boundaries', select_ugid=su) self.assertEqual(len(list(g.value)), 3) geoms = [{'geom': geom, 'properties': {'UGID': 1}}, {'geom': geom, 'properties': {'UGID': 2}}] Geom(geoms) bbox = [-120, 40, -110, 50] g = Geom(bbox) self.assertEqual(g.value[0].geom.polygon.value[0, 0].bounds, tuple(map(float, bbox))) sui = GeomUid('ID') g = Geom(bbox, geom_uid=sui) self.assertEqual(g.geom_uid, 'ID') g = Geom(bbox, geom_uid='ID') self.assertEqual(g.geom_uid, 'ID') ################################################################################################################ # tests for geom_select_sql_where g = Geom('state_boundaries') self.assertIsNone(g.geom_select_sql_where) s = 'STATE_NAME in ("Wisconsin", "Vermont")' ws = [GeomSelectSqlWhere(s), s] for w in ws: g = Geom('state_boundaries', geom_select_sql_where=w) self.assertEqual(g.geom_select_sql_where, s)
def test_geom(self): geom = make_poly((37.762,38.222),(-102.281,-101.754)) g = Geom(geom) self.assertEqual(type(g.value),GeometryDataset) g.value = None self.assertEqual(None,g.value) g = Geom(None) self.assertEqual(g.value,None) self.assertEqual(str(g),'geom=None') g = Geom('-120|40|-110|50') self.assertEqual(g.value.spatial.geom[0].bounds,(-120.0, 40.0, -110.0, 50.0)) self.assertEqual(str(g),'geom=-120.0|40.0|-110.0|50.0') g = Geom('mi_watersheds') self.assertEqual(str(g),'geom=mi_watersheds') geoms = ShpCabinet().get_geoms('mi_watersheds') g = Geom('mi_watersheds') self.assertEqual(len(g.value),len(geoms)) su = SelectUgid([1,2,3]) g = Geom('mi_watersheds',select_ugid=su) self.assertEqual(len(g.value),3) geoms = GeometryDataset(uid=[1,2],geom=[geom,geom]) g = Geom(geoms) bbox = [-120,40,-110,50] g = Geom(bbox) self.assertEqual(g.value.spatial.geom[0].bounds,tuple(map(float,bbox)))
def intersects(self,polygon): ## reset the values to ensure mask is properly applied self._value = None ## reset the weights self._weights = None ## do the initial grid subset grid = self.grid.subset(polygon=polygon) ## construct the spatial index index_grid = si.build_index_grid(30.0,polygon) index = si.build_index(polygon,index_grid) ## the fill arrays geom = np.ones(grid.shape,dtype=object) geom = np.ma.array(geom,mask=True) ## loop performing the spatial operation row = grid.row.bounds col = grid.column.bounds index_intersects = si.index_intersects geom_mask = geom.mask for ii,jj in product(range(row.shape[0]),range(col.shape[0])): rref = row[ii,:] cref = col[jj,:] test_geom = make_poly(rref,cref) geom[ii,jj] = test_geom if index_intersects(test_geom,index): geom_mask[ii,jj] = False else: geom_mask[ii,jj] = True ret = self.__class__(grid=grid,geom=geom,uid=grid.uid) return(ret)
def test_geom(self): geom = make_poly((37.762,38.222),(-102.281,-101.754)) g = Geom(geom) self.assertEqual(type(g.value),list) g.value = None self.assertEqual(None,g.value) g = Geom(None) self.assertEqual(g.value,None) self.assertEqual(str(g),'geom=None') g = Geom('-120|40|-110|50') self.assertEqual(g.value[0]['geom'].bounds,(-120.0, 40.0, -110.0, 50.0)) self.assertEqual(str(g),'geom=-120.0|40.0|-110.0|50.0') g = Geom('mi_watersheds') self.assertEqual(str(g),'geom=mi_watersheds') geoms = list(ShpCabinetIterator('mi_watersheds')) g = Geom('mi_watersheds') self.assertEqual(len(list(g.value)),len(geoms)) su = SelectUgid([1,2,3]) g = Geom('mi_watersheds',select_ugid=su) self.assertEqual(len(list(g.value)),3) geoms = [{'geom':geom,'properties':{'UGID':1}},{'geom':geom,'properties':{'UGID':2}}] g = Geom(geoms) bbox = [-120,40,-110,50] g = Geom(bbox) self.assertEqual(g.value[0]['geom'].bounds,tuple(map(float,bbox)))
def test_get_intersects_domain_polygon(self): regular = make_poly((36.61, 41.39), (-101.41, -95.47)) field = self.get_field(with_value=True) for b in [True, False]: ret = field.get_intersects(regular, use_spatial_index=b) self.assertNumpyAll(ret.variables['tmax'].value, field.variables['tmax'].value) self.assertNumpyAll(field.spatial.grid.value, ret.spatial.grid.value)
def shapely_grid(dim,rtup,ctup,target=None): if dim is None: ## construct an average of 10 polygons row_dim = np.ceil(np.abs(rtup[0]-rtup[1])/5.0) col_dim = np.ceil(np.abs(ctup[0]-ctup[1])/5.0) dim = np.mean([row_dim,col_dim]) row_bounds = np.arange(rtup[0],rtup[1]+dim,dim) min_row = row_bounds[0:-1] max_row = row_bounds[1:] row_bounds = np.hstack((min_row.reshape(-1,1),max_row.reshape(-1,1))) col_bounds = np.arange(ctup[0],ctup[1]+dim,dim) min_col = col_bounds[0:-1] max_col = col_bounds[1:] col_bounds = np.hstack((min_col.reshape(-1,1),max_col.reshape(-1,1))) polygons = [] for ii in range(row_bounds.shape[0]): rtup = (row_bounds[ii,0],row_bounds[ii,1]) for jj in range(col_bounds.shape[0]): ctup = (col_bounds[jj,0],col_bounds[jj,1]) polygon = make_poly(rtup,ctup) if target is not None and keep(target,polygon): polygons.append(polygon) elif target is None: polygons.append(polygon) return(MultiPolygon(polygons))
def test_spatial_dimension(self): rd = self.test_data.get_rd('cancm4_tas') ds = nc.Dataset(rd.uri,'r') row_data = ds.variables['lat'][:] row_bounds = ds.variables['lat_bnds'][:] col_data = ds.variables['lon'][:] col_bounds = ds.variables['lon_bnds'][:] rd = NcRowDimension(value=row_data,bounds=row_bounds) cd = NcColumnDimension(value=col_data,bounds=col_bounds) gd = NcGridDimension(row=rd,column=cd) self.assertEqual(gd.resolution,2.8009135133922354) sd = NcSpatialDimension(row=rd,column=cd) sgd = gd.subset() self.assertEqual(sgd.shape,(rd.shape[0],cd.shape[0])) poly = make_poly((-62,59),(87,244)) sgd = gd.subset(polygon=poly) self.assertEqual(sgd.uid.shape,(sgd.row.shape[0],sgd.column.shape[0])) self.assertTrue(sum(sgd.shape) < sum(gd.shape)) lgd = gd[0:5,0:5] self.assertEqual(lgd.shape,(5,5)) vd = NcPolygonDimension(gd) self.assertEqual(vd.geom.shape,vd.grid.shape) ivd = vd.intersects(poly) self.assertTrue(sum(ivd.geom.shape) < sum(vd.geom.shape)) self.assertEqual(ivd.weights.max(),1.0) cvd = vd.clip(poly) self.assertEqual(ivd.shape,cvd.shape) self.assertFalse(ivd.weights.sum() == cvd.weights.sum()) ds.close()
def intersects(self, polygon): ## reset the values to ensure mask is properly applied self._value = None ## reset the weights self._weights = None ## do the initial grid subset grid = self.grid.subset(polygon=polygon) ## construct the spatial index index_grid = si.build_index_grid(30.0, polygon) index = si.build_index(polygon, index_grid) ## the fill arrays geom = np.ones(grid.shape, dtype=object) geom = np.ma.array(geom, mask=True) ## loop performing the spatial operation row = grid.row.bounds col = grid.column.bounds index_intersects = si.index_intersects geom_mask = geom.mask for ii, jj in product(range(row.shape[0]), range(col.shape[0])): rref = row[ii, :] cref = col[jj, :] test_geom = make_poly(rref, cref) geom[ii, jj] = test_geom if index_intersects(test_geom, index): geom_mask[ii, jj] = False else: geom_mask[ii, jj] = True ret = self.__class__(grid=grid, geom=geom, uid=grid.uid) return (ret)
def test_empty_intersection(self): geom = make_poly((20,25),(-90,-80)) with self.assertRaises(exc.ExtentError): self.get_ret(kwds={'geom':geom}) ret = self.get_ret(kwds={'geom':geom,'allow_empty':True}) self.assertEqual(len(ret[1].variables),0)
def test_empty_intersection(self): geom = make_poly((20, 25), (-90, -80)) with self.assertRaises(exc.ExtentError): self.get_ret(kwds={'geom': geom}) ret = self.get_ret(kwds={'geom': geom, 'allow_empty': True}) self.assertEqual(len(ret[1].variables), 0)
def test_get_intersects_polygon_all(self): for b in [True,False]: sdim = self.get_sdim(bounds=b) poly = make_poly((37,41),(-101,-96)) ret = sdim.get_intersects(poly) self.assertNumpyAll(sdim.grid.value,ret.grid.value) self.assertNumpyAll(sdim.grid.value.mask[0,:,:],sdim.geom.point.value.mask) self.assertEqual(ret.shape,(3,4))
def test_csv_conversion(self): ops = OcgOperations(dataset=self.get_dataset(),output_format='csv') ret = self.get_ret(ops) ## test with a geometry to check writing of user-geometry overview shapefile geom = make_poly((38,39),(-104,-103)) ops = OcgOperations(dataset=self.get_dataset(),output_format='csv',geom=geom) ret = ops.execute()
def test_iter_regridded_fields_different_grid_shapes(self): """Test regridding a downscaled dataset to GCM output. The input and output grids have different shapes.""" downscaled = self.test_data.get_rd("maurer_2010_tas") downscaled.time_region = {"month": [2], "year": [1990]} downscaled = downscaled.get() poly = make_poly([37, 43], [-104, -94]) downscaled = downscaled.get_intersects(poly) downscaled.spatial.unwrap() downscaled.spatial.crs = Spherical() gcm = self.test_data.get_rd("cancm4_tas") gcm = gcm.get() poly = make_poly([37, 43], [-104 + 360, -94 + 360]) gcm = gcm.get_intersects(poly) gcm.spatial.crs = Spherical() # add masked values to the source and destination self.assertFalse(downscaled.spatial.get_mask().any()) self.assertFalse(gcm.spatial.get_mask().any()) mask = gcm.spatial.get_mask() mask[1, 3] = True gcm.spatial.set_mask(mask) desired = { "std": 3.1385237308556095, "trace": -11.13192056119442, "min": -11.858446, "max": 9.8645229, "shape": (1, 28, 1, 3, 5), "mean": 0.047387103645169008, } for regridded in iter_regridded_fields([downscaled], gcm): self.assertEqual(regridded.shape, (1, 28, 1, 3, 5)) self.assertEqual(regridded.variables.keys(), ["tas"]) self.assertDescriptivesAlmostEqual(desired, regridded.variables["tas"].value) self.assertNumpyAll(gcm.spatial.get_mask(), mask) for variable in regridded.variables.itervalues(): vmask = variable.value.mask self.assertTrue(vmask[:, :, :, 1, 3].all()) self.assertEqual(vmask.sum(), 28)
def test_get_intersects_polygon_no_point_overlap(self): for b in [True,False]: sdim = self.get_sdim(bounds=b) poly = make_poly((39.25,39.75),(-97.75,-97.25)) if b is False: with self.assertRaises(EmptySubsetError): sdim.get_intersects(poly) else: ret = sdim.get_intersects(poly) self.assertEqual(ret.shape,(2,2))
def test_get_wrapped_state_from_geometry(self): geoms = [ Point(-130, 40), MultiPoint([Point(-130, 40), Point(30, 50)]), make_poly((30, 40), (-130, -120)), MultiPolygon([ make_poly((30, 40), (-130, -120)), make_poly((30, 40), (130, 160)) ]) ] c = CoordinateReferenceSystem for geom in geoms: ret = c._get_wrapped_state_from_geometry_(geom) self.assertEqual(ret, WrappedState.WRAPPED) pt = Point(270, 50) ret = c._get_wrapped_state_from_geometry_(pt) self.assertEqual(ret, WrappedState.UNWRAPPED)
def test_spatial(self): geom = make_poly((38,39),(-93,-92)) for abstraction in ['polygon','point']: ret = self.get_ret(kwds={'geom':geom,'abstraction':abstraction}) self.assertEqual(len(ret[1].variables[self.var].spatial.vector.uid.compressed()),4) self.get_ret(kwds={'vector_wrap':False}) ret = self.get_ret(kwds={'geom':geom,'vector_wrap':False,'abstraction':abstraction}) self.assertEqual(len(ret[1].variables[self.var].spatial.vector.uid.compressed()),4)
def test_get_intersects_domain_polygon(self): regular = make_poly((36.61,41.39),(-101.41,-95.47)) field = self.get_field(with_value=True) for b in [True,False]: try: ret = field.get_intersects(regular,use_spatial_index=b) self.assertNumpyAll(ret.variables['tmax'].value,field.variables['tmax'].value) self.assertNumpyAll(field.spatial.grid.value,ret.spatial.grid.value) except ImportError: with self.assertRaises(ImportError): import_module('rtree')
def test_csv_conversion(self): ocgis.env.OVERWRITE = True ops = OcgOperations(dataset=self.get_dataset(), output_format='csv') ret = self.get_ret(ops) ## test with a geometry to check writing of user-geometry overview shapefile geom = make_poly((38, 39), (-104, -103)) ops = OcgOperations(dataset=self.get_dataset(), output_format='csv', geom=geom) ret = ops.execute()
def test_spatial(self): ## intersects geom = make_poly((37.5, 39.5), (-104.5, -102.5)) ret = self.get_ret(kwds={'geom': geom}) ref = ret[1] gids = set([6, 7, 10, 11]) ret_gids = set(ref.variables[self.var].spatial.vector.uid.compressed()) intersection = gids.intersection(ret_gids) self.assertEqual(len(intersection), 4) self.assertTrue( np.all(ref.variables[self.var].value[0, 0, :, :] == np.array( [[1.0, 2.0], [3.0, 4.0]]))) ## intersection geom = make_poly((38, 39), (-104, -103)) ret = self.get_ret(kwds={'geom': geom, 'spatial_operation': 'clip'}) self.assertEqual( len(ret[1].variables[self.var].spatial.vector.uid.compressed()), 4) self.assertEqual(ret[1].variables[self.var].value.shape, (61, 2, 2, 2)) ref = ret[1].variables[self.var].value self.assertTrue( np.all(ref[0, 0, :, :] == np.array([[1, 2], [3, 4]], dtype=float))) ## compare areas to intersects returns ref = ret[1].variables[self.var] intersection_areas = [g.area for g in ref.spatial.vector.geom.flat] for ii in intersection_areas: self.assertAlmostEqual(ii, 0.25) ## intersection + aggregation geom = make_poly((38, 39), (-104, -103)) ret = self.get_ret(kwds={ 'geom': geom, 'spatial_operation': 'clip', 'aggregate': True }) ref = ret[1] self.assertEqual( len(ref.variables[self.var].spatial.vector.uid.flatten()), 1) self.assertEqual( ref.variables[self.var].spatial.vector.geom.flatten()[0].area, 1.0) self.assertEqual(ref.variables[self.var].value.flatten().mean(), 2.5)
def test_get_clip(self): sdim = self.get_sdim(bounds=True) poly = make_poly((37.75,38.25),(-100.25,-99.75)) ret = sdim.get_clip(poly) self.assertEqual(ret.uid,np.array([[9]])) self.assertTrue(poly.almost_equals(ret.geom.polygon.value[0,0])) self.assertNumpyAll(ret.geom.point.value.shape,ret.geom.polygon.shape) ref_pt = ret.geom.point.value[0,0] ref_poly = ret.geom.polygon.value[0,0] self.assertTrue(ref_poly.intersects(ref_pt))
def test_empty_intersection(self): geom = make_poly((20,25),(-90,-80)) geom = [{'ugid':1,'geom':geom}] with self.assertRaises(exc.ExtentError): self.get_ret(kwds={'geom':geom}) ret = self.get_ret(kwds={'geom':geom,'allow_empty':True}) ref = ret[1].variables[self.var].spatial.uid self.assertEqual(len(ref),0) ref = ret[1] self.assertTrue(ref.is_empty)
def _get_all_geoms_(self): ## the fill arrays geom = np.ones(self.grid.shape,dtype=object) geom = np.ma.array(geom,mask=False) ## loop performing the spatial operation row = self.grid.row.bounds col = self.grid.column.bounds for ii,jj in product(range(row.shape[0]),range(col.shape[0])): rref = row[ii,:] cref = col[jj,:] geom[ii,jj] = make_poly(rref,cref) return(geom)
def test_get_intersects_polygon_all(self): for b in [True,False]: sdim = self.get_sdim(bounds=b) poly = make_poly((37,41),(-101,-96)) for u in [True,False]: try: ret = sdim.get_intersects(poly,use_spatial_index=u) self.assertNumpyAll(sdim.grid.value,ret.grid.value) self.assertNumpyAll(sdim.grid.value.mask[0,:,:],sdim.geom.point.value.mask) self.assertEqual(ret.shape,(3,4)) except ImportError: with self.assertRaises(ImportError): import_module('rtree')
def wrap_var(var): right_clip = make_poly((-90,90),(180,360)) left_clip = make_poly((-90,90),(-180,180)) def _shift_(geom): try: coords = np.array(geom.exterior.coords) coords[:,0] = coords[:,0] - 360 ret = Polygon(coords) except AttributeError: polygons = np.empty(len(geom),dtype=object) for ii,polygon in enumerate(geom): coords = np.array(polygon.exterior.coords) coords[:,0] = coords[:,0] - 360 polygons[ii] = Polygon(coords) ret = MultiPolygon(polygons) return(ret) geoms = var.spatial._value if not isinstance(geoms[0,0],Point): for idx,geom in iter_array(geoms,return_value=True): bounds = np.array(geom.bounds) if np.all([bounds[0] > 180,bounds[2] > 180]): geoms[idx] = _shift_(geom) elif bounds[1] <= 180 and bounds[2] > 180: left = [poly for poly in _get_iter_(geom.intersection(left_clip))] right = [poly for poly in _get_iter_(_shift_(geom.intersection(right_clip)))] try: geoms[idx] = MultiPolygon(left+right) except TypeError: left = filter(lambda x: type(x) != LineString,left) right = filter(lambda x: type(x) != LineString,right) geoms[idx] = MultiPolygon(left+right) else: continue else: for idx,geom in iter_array(geoms,return_value=True): if geom.x > 180: geoms[idx] = Point(geom.x-360,geom.y)
def _get_all_geoms_(self): ## the fill arrays geom = np.ones(self.grid.shape, dtype=object) geom = np.ma.array(geom, mask=False) ## loop performing the spatial operation row = self.grid.row.bounds col = self.grid.column.bounds for ii, jj in product(range(row.shape[0]), range(col.shape[0])): rref = row[ii, :] cref = col[jj, :] geom[ii, jj] = make_poly(rref, cref) return (geom)
def test_iter_regridded_fields_different_grid_shapes(self): """Test regridding a downscaled dataset to GCM output. The input and output grids have different shapes.""" downscaled = self.test_data.get_rd("maurer_2010_tas") downscaled.time_region = {"month": [2], "year": [1990]} downscaled = downscaled.get() poly = make_poly([37, 43], [-104, -94]) downscaled = downscaled.get_intersects(poly) downscaled.spatial.unwrap() downscaled.spatial.crs = Spherical() gcm = self.test_data.get_rd("cancm4_tas") gcm = gcm.get() poly = make_poly([37, 43], [-104 + 360, -94 + 360]) gcm = gcm.get_intersects(poly) gcm.spatial.crs = Spherical() # add masked values to the source and destination self.assertFalse(downscaled.spatial.get_mask().any()) self.assertFalse(gcm.spatial.get_mask().any()) mask = gcm.spatial.get_mask() mask[1, 3] = True gcm.spatial.set_mask(mask) dmask = downscaled.spatial.get_mask() dmask[:] = True downscaled.spatial.set_mask(dmask) downscaled.variables.first().value.mask[:] = True for regridded in iter_regridded_fields([downscaled], gcm): self.assertEqual(regridded.shape, (1, 28, 1, 3, 5)) self.assertEqual(regridded.variables.keys(), ["tas"]) self.assertAlmostEqual(regridded.variables["tas"].value.data.mean(), 0.057409391) self.assertNumpyAll(gcm.spatial.get_mask(), mask) for variable in regridded.variables.itervalues(): vmask = variable.value.mask self.assertTrue(vmask[:, :, :, 1, 3].all()) self.assertEqual(vmask.sum(), 28)
def assertGridCorners(self, grid): """ :type grid: :class:`ocgis.new_interface.grid.Grid` """ assert grid.corners is not None def _get_is_ascending_(arr): """ Return ``True`` if the array is ascending from index 0 to -1. :type arr: :class:`numpy.ndarray` :rtype: bool """ assert arr.ndim == 1 if arr[0] < arr[-1]: ret = True else: ret = False return ret # Assert polygon constructed from grid corners contains the associated centroid value. for ii, jj in itertools.product(list(range(grid.shape[0])), list(range(grid.shape[1]))): pt = Point(grid.get_value().data[1, ii, jj], grid.get_value().data[0, ii, jj]) poly_corners = grid.corners.data[:, ii, jj] rtup = (poly_corners[0, :].min(), poly_corners[0, :].max()) ctup = (poly_corners[1, :].min(), poly_corners[1, :].max()) poly = make_poly(rtup, ctup) self.assertTrue(poly.contains(pt)) # Assert masks are equivalent between value and corners. for (ii, jj), m in iter_array(grid.get_value().mask[0, :, :], return_value=True): if m: self.assertTrue(grid.corners.mask[:, ii, jj].all()) else: self.assertFalse(grid.corners.mask[:, ii, jj].any()) grid_y = grid._y grid_x = grid._x if grid_y is not None or grid_x is not None: self.assertEqual( _get_is_ascending_(grid_y.get_value()), _get_is_ascending_(grid.corners.data[0, :, 0][:, 0])) self.assertEqual( _get_is_ascending_(grid_x.get_value()), _get_is_ascending_(grid.corners.data[1, 0, :][:, 0]))
def test_get_intersects_polygon_no_point_overlap(self): for b in [True,False]: sdim = self.get_sdim(bounds=b) poly = make_poly((39.25,39.75),(-97.75,-97.25)) for u in [True,False]: try: if b is False: with self.assertRaises(EmptySubsetError): sdim.get_intersects(poly,use_spatial_index=u) else: ret = sdim.get_intersects(poly,use_spatial_index=u) self.assertEqual(ret.shape,(2,2)) except ImportError: with self.assertRaises(ImportError): import_module('rtree')
def test_get_intersects_polygon_small(self): for b in [True,False]: sdim = self.get_sdim(bounds=b) poly = make_poly((37.75,38.25),(-100.25,-99.75)) ret = sdim.get_intersects(poly) to_test = np.ma.array([[[38]],[[-100]]],mask=False) self.assertNumpyAll(ret.grid.value,to_test) self.assertNumpyAll(ret.uid,np.array([[9]])) self.assertEqual(ret.shape,(1,1)) to_test = ret.geom.point.value.compressed()[0] self.assertTrue(to_test.almost_equals(Point(-100,38))) if b is False: with self.assertRaises(ImproperPolygonBoundsError): ret.geom.polygon else: to_test = ret.geom.polygon.value.compressed()[0].bounds self.assertEqual((-100.5,37.5,-99.5,38.5),to_test)
def test_geom(self): geom = make_poly((37.762,38.222),(-102.281,-101.754)) g = definition.Geom(geom) self.assertEqual(type(g.value),list) self.assertEqual(g.value[0]['geom'].bounds,(-102.281, 37.762, -101.754, 38.222)) g = definition.Geom(None) self.assertEqual(g.value,None) self.assertEqual(str(g),'geom=None') g = definition.Geom('mi_watersheds') self.assertEqual(str(g),'geom="mi_watersheds"') geoms = ShpCabinetIterator('mi_watersheds') g = definition.Geom(geoms) self.assertEqual(len(list(g.value)),60) self.assertEqual(g._shp_key,'mi_watersheds')
def test_keyword_geom(self): geom = make_poly((37.762, 38.222), (-102.281, -101.754)) g = definition.Geom(geom) self.assertEqual(type(g.value), tuple) self.assertEqual(g.value[0].geom.get_value()[0].bounds, (-102.281, 37.762, -101.754, 38.222)) g = definition.Geom(None) self.assertEqual(g.value, None) self.assertEqual(str(g), 'geom=None') g = definition.Geom('mi_watersheds') self.assertEqual(str(g), 'geom="mi_watersheds"') geoms = GeomCabinetIterator('mi_watersheds') g = definition.Geom(geoms) self.assertEqual(len(list(g.value)), 60) self.assertEqual(g._shp_key, 'mi_watersheds')
def test_geom(self): geom = make_poly((37.762,38.222),(-102.281,-101.754)) g = definition.Geom(geom) self.assertEqual(type(g.value),GeometryDataset) g = definition.Geom(None) self.assertEqual(g.value,None) self.assertEqual(str(g),'geom=None') g = definition.Geom('-120|40|-110|50') self.assertEqual(str(g),'geom=-120.0|40.0|-110.0|50.0') g = definition.Geom('mi_watersheds') self.assertEqual(str(g),'geom=mi_watersheds') geoms = ShpDataset('mi_watersheds') g = definition.Geom(geoms) self.assertEqual(len(g.value),60)
def test_geom(self): geom = make_poly((37.762, 38.222), (-102.281, -101.754)) g = definition.Geom(geom) self.assertEqual(type(g.value), GeometryDataset) g = definition.Geom(None) self.assertEqual(g.value, None) self.assertEqual(str(g), 'geom=None') g = definition.Geom('-120|40|-110|50') self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0') g = definition.Geom('mi_watersheds') self.assertEqual(str(g), 'geom=mi_watersheds') geoms = ShpDataset('mi_watersheds') g = definition.Geom(geoms) self.assertEqual(len(g.value), 60)
def test_spatial(self): geom = make_poly((38, 39), (-93, -92)) for abstraction in ['polygon', 'point']: ret = self.get_ret(kwds={'geom': geom, 'abstraction': abstraction}) self.assertEqual( len(ret[1].variables[ self.var].spatial.vector.uid.compressed()), 4) self.get_ret(kwds={'vector_wrap': False}) ret = self.get_ret(kwds={ 'geom': geom, 'vector_wrap': False, 'abstraction': abstraction }) self.assertEqual( len(ret[1].variables[ self.var].spatial.vector.uid.compressed()), 4)
def test_abstraction_point(self): rd = self.test_data.get_rd('cancm4_tas') ods = NcDataset(request_dataset=rd, abstraction='point') with self.assertRaises(AttributeError): ods.spatial.shape self.assertTrue(isinstance(ods.spatial.vector, NcPointDimension)) geom = ods.spatial.vector.geom self.assertEqual(geom.shape, (64, 128)) self.assertTrue(isinstance(geom[0, 0], Point)) weights = ods.spatial.vector.weights self.assertEqual(weights.max(), 1.) poly = make_poly((-62, 59), (87, 244)) nods = ods.get_subset(spatial_operation='intersects', igeom=poly) with self.assertRaises(AttributeError): nods.spatial.shape self.assertNotEqual(ods.spatial.vector.shape, nods.spatial.vector.shape) nods.aggregate() self.assertEqual(nods.spatial.vector.geom.shape, (1, 1)) self.assertTrue(isinstance(nods.spatial.vector.geom[0, 0], Point))
def shapely_grid(dim, rtup, ctup, target=None): row_bounds = np.arange(rtup[0], rtup[1] + dim, dim) min_row = row_bounds[0:-1] max_row = row_bounds[1:] row_bounds = np.hstack((min_row.reshape(-1, 1), max_row.reshape(-1, 1))) col_bounds = np.arange(ctup[0], ctup[1] + dim, dim) min_col = col_bounds[0:-1] max_col = col_bounds[1:] col_bounds = np.hstack((min_col.reshape(-1, 1), max_col.reshape(-1, 1))) polygons = [] for ii in range(row_bounds.shape[0]): rtup = (row_bounds[ii, 0], row_bounds[ii, 1]) for jj in range(col_bounds.shape[0]): ctup = (col_bounds[jj, 0], col_bounds[jj, 1]) polygon = make_poly(rtup, ctup) if target is not None and keep(target, polygon): polygons.append(polygon) elif target is None: polygons.append(polygon) return (MultiPolygon(polygons))
def test_spatial_dimension(self): rd = self.test_data.get_rd('cancm4_tas') ds = nc.Dataset(rd.uri, 'r') row_data = ds.variables['lat'][:] row_bounds = ds.variables['lat_bnds'][:] col_data = ds.variables['lon'][:] col_bounds = ds.variables['lon_bnds'][:] rd = NcRowDimension(value=row_data, bounds=row_bounds) cd = NcColumnDimension(value=col_data, bounds=col_bounds) gd = NcGridDimension(row=rd, column=cd) self.assertEqual(gd.resolution, 2.8009135133922354) sd = NcSpatialDimension(row=rd, column=cd) sgd = gd.subset() self.assertEqual(sgd.shape, (rd.shape[0], cd.shape[0])) poly = make_poly((-62, 59), (87, 244)) sgd = gd.subset(polygon=poly) self.assertEqual(sgd.uid.shape, (sgd.row.shape[0], sgd.column.shape[0])) self.assertTrue(sum(sgd.shape) < sum(gd.shape)) lgd = gd[0:5, 0:5] self.assertEqual(lgd.shape, (5, 5)) vd = NcPolygonDimension(gd) self.assertEqual(vd.geom.shape, vd.grid.shape) ivd = vd.intersects(poly) self.assertTrue(sum(ivd.geom.shape) < sum(vd.geom.shape)) self.assertEqual(ivd.weights.max(), 1.0) cvd = vd.clip(poly) self.assertEqual(ivd.shape, cvd.shape) self.assertFalse(ivd.weights.sum() == cvd.weights.sum()) ds.close()
def iter_operations(self, start=0): datasets = { 1: { 'uri': '/usr/local/climate_data/CanCM4/tasmax_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc', 'variable': 'tasmax', 'alias': 'tasmax' }, 2: { 'uri': '/usr/local/climate_data/CanCM4/tasmin_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc', 'variable': 'tasmin', 'alias': 'tasmin' } } output_format = { 'output_format': [ # 'shp', 'keyed', 'meta', 'nc', 'csv' ] } snippet = { 'snippet': [ True, # False ] } dataset = {'dataset': [[1], [1, 2]]} geom = { 'geom': [ self.alaska, None, self.california, self.state_boundaries, [{ 'ugid': 1, 'geom': make_poly((24.2, 50.8), (-128.7, -65.2)) }], # self.world_countries ] } aggregate = {'aggregate': [True, False]} spatial_operation = { 'spatial_operation': [ 'clip', 'intersects', ] } vector_wrap = {'vector_wrap': [True, False]} abstraction = {'abstraction': ['polygon', 'point']} agg_selection = {'agg_selection': [True, False]} level_range = {'level_range': [None, [1, 1]]} time_range = { 'time_range': [[datetime(2001, 1, 1), datetime(2001, 12, 31, 23, 59, 59)], None] } allow_empty = {'allow_empty': [True, False]} calc = { 'calc': [ # [{'func':'mean','name':'my_mean'}], None, ] } calc_grouping = {'calc_grouping': [['month', 'year'], ['year']]} args = [ output_format, snippet, dataset, geom, aggregate, spatial_operation, vector_wrap, abstraction, agg_selection, level_range, time_range, allow_empty, calc, calc_grouping ] combined = OrderedDict() for arg in args: combined.update(arg) for ii, ret in enumerate(itertools.product(*combined.values())): if ii >= start: kwds = deepcopy(dict(zip(combined.keys(), ret))) time_range = kwds.pop('time_range') level_range = kwds.pop('level_range') rds = [ RequestDataset(datasets[jj]['uri'], datasets[jj]['variable'], time_range=time_range, level_range=level_range) for jj in kwds['dataset'] ] kwds['dataset'] = rds tmp_path = tempfile.mkdtemp(prefix='ocgis') os.rmdir(tmp_path) kwds['prefix'] = os.path.split(tmp_path)[1] ops = OcgOperations(**kwds) yield (ii, ops)
def __init__(self, axis=0.0): self.axis = float(axis) self.right_clip = make_poly((-90, 90), (180, 360)) self.left_clip = make_poly((-90, 90), (-180, 180)) self.clip1 = make_poly((-90, 90), (-180, axis)) self.clip2 = make_poly((-90, 90), (axis, 180))
def test_load(self): poly = make_poly((-62, 59), (87, 244)) gds = GeometryDataset(uid=[1], geom=[poly])
def test_empty_mask(self): geom = make_poly((37.762, 38.222), (-102.281, -101.754)) with self.assertRaises(exc.MaskedDataError): ret = self.get_ret(kwds={'geom': geom}) ret = self.get_ret(kwds={'geom': geom, 'allow_empty': True})
def test_init(self): geom = make_poly((37.762, 38.222), (-102.281, -101.754)) g = Geom(geom) self.assertEqual(type(g.value), tuple) g.value = None self.assertEqual(None, g.value) g = Geom(None) self.assertEqual(g.value, None) self.assertEqual(str(g), 'geom=None') g = Geom('-120|40|-110|50') self.assertEqual(g.value[0].geom.get_value()[0].bounds, (-120.0, 40.0, -110.0, 50.0)) self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0') ocgis.env.DIR_GEOMCABINET = self.path_bin g = Geom('state_boundaries') self.assertEqual(str(g), 'geom="state_boundaries"') geoms = list(GeomCabinetIterator('state_boundaries')) g = Geom('state_boundaries') self.assertEqual(len(list(g.value)), len(geoms)) sci = GeomCabinetIterator(key='state_boundaries') self.assertFalse(sci.as_field) g = Geom(sci) for _ in range(2): for ii, element in enumerate(g.value): self.assertIsInstance(element, Field) self.assertGreater(ii, 10) su = GeomSelectUid([1, 2, 3]) g = Geom('state_boundaries', select_ugid=su) self.assertEqual(len(list(g.value)), 3) geoms = [{ 'geom': geom, 'properties': { 'UGID': 1 } }, { 'geom': geom, 'properties': { 'UGID': 2 } }] Geom(geoms) bbox = [-120, 40, -110, 50] g = Geom(bbox) self.assertEqual(g.value[0].geom.get_value()[0].bounds, tuple(map(float, bbox))) sui = GeomUid('ID') g = Geom(bbox, geom_uid=sui) self.assertEqual(g.geom_uid, 'ID') g = Geom(bbox, geom_uid='ID') self.assertEqual(g.geom_uid, 'ID') # Tests for geom_select_sql_where ############################################################################## g = Geom('state_boundaries') self.assertIsNone(g.geom_select_sql_where) s = 'STATE_NAME in ("Wisconsin", "Vermont")' ws = [GeomSelectSqlWhere(s), s] for w in ws: g = Geom('state_boundaries', geom_select_sql_where=w) self.assertEqual(g.geom_select_sql_where, s) # Test passing a folder which is not allowed ################################################################### with self.assertRaises(DefinitionValidationError): Geom(tempfile.gettempdir())