def test_get_buffered_subset_sdim(self): proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs' buffer_crs_list = [None, CoordinateReferenceSystem(proj4=proj4)] poly = make_poly((36, 44), (-104, -95)) for buffer_crs in buffer_crs_list: subset_sdim = SpatialDimension.from_records([{'geom': poly, 'properties': {'UGID': 1}}], crs=CFWGS84()) self.assertEqual(subset_sdim.crs, CFWGS84()) if buffer_crs is None: buffer_value = 1 else: buffer_value = 10 ret = SpatialSubsetOperation._get_buffered_subset_sdim_(subset_sdim, buffer_value, buffer_crs=buffer_crs) ref = ret.geom.polygon.value[0, 0] if buffer_crs is None: self.assertEqual(ref.bounds, (-105.0, 35.0, -94.0, 45.0)) else: self.assertNumpyAllClose(np.array(ref.bounds), np.array((-104.00013263459613, 35.9999147913708, -94.99986736540386, 44.00008450528758))) self.assertEqual(subset_sdim.crs, ret.crs) # check deepcopy ret.geom.polygon.value[0, 0] = make_poly((1, 2), (3, 4)) ref_buffered = ret.geom.polygon.value[0, 0] ref_original = subset_sdim.geom.polygon.value[0, 0] with self.assertRaises(AssertionError): self.assertNumpyAllClose(np.array(ref_buffered.bounds), np.array(ref_original.bounds))
def iter_geoms(self, key=None, select_uid=None, path=None, load_geoms=True, as_spatial_dimension=False, uid=None, select_sql_where=None): """ See documentation for :class:`~ocgis.util.shp_cabinet.ShpCabinetIterator`. """ # ensure select ugid is in ascending order if select_uid is not None: test_select_ugid = list(deepcopy(select_uid)) test_select_ugid.sort() if test_select_ugid != list(select_uid): raise ValueError('"select_uid" must be sorted in ascending order.') # get the path to the output shapefile shp_path = self._get_path_by_key_or_direct_path_(key=key, path=path) # get the source CRS meta = self.get_meta(path=shp_path) # open the target shapefile ds = ogr.Open(shp_path) try: # return the features iterator features = self._get_features_object_(ds, uid=uid, select_uid=select_uid, select_sql_where=select_sql_where) build = True for ctr, feature in enumerate(features): if load_geoms: yld = {'geom': wkb.loads(feature.geometry().ExportToWkb())} else: yld = {} items = feature.items() properties = OrderedDict([(key, items[key]) for key in feature.keys()]) yld.update({'properties': properties, 'meta': meta}) if build: uid, add_uid = get_uid_from_properties(properties, uid) build = False # add the unique identifier if required if add_uid: properties[uid] = ctr + 1 # ensure the unique identifier is an integer else: properties[uid] = int(properties[uid]) if as_spatial_dimension: yld = SpatialDimension.from_records([yld], crs=yld['meta']['crs'], uid=uid) yield yld try: assert ctr >= 0 except UnboundLocalError: # occurs if there were not feature returned by the iterator. raise a more clear exception. msg = 'No features returned from target shapefile. Were features appropriately selected?' raise ValueError(msg) finally: # close the dataset object ds.Destroy() ds = None
def test_get_spatial_subset_rotated_pole(self): """Test input has rotated pole with now output CRS.""" rd = self.rd_rotated_pole ss = SpatialSubsetOperation(rd) subset_sdim = SpatialDimension.from_records([self.germany]) ret = ss.get_spatial_subset('intersects', subset_sdim) self.assertEqual(ret.spatial.crs, rd.get().spatial.crs) self.assertAlmostEqual(ret.spatial.grid.value.data.mean(), -2.0600000000000009)
def test_spatial_dimension(self): """Test using a SpatialDimension as input value.""" sdim = SpatialDimension.from_records(GeomCabinetIterator(key='state_boundaries')) self.assertIsInstance(sdim.crs, CFWGS84) g = Geom(sdim) self.assertEqual(len(g.value), 51) for sdim in g.value: self.assertIsInstance(sdim, SpatialDimension) self.assertEqual(sdim.shape, (1, 1))
def parse(self, value): if type(value) in [list, tuple]: if all([isinstance(element, dict) for element in value]): for ii, element in enumerate(value, start=1): if 'geom' not in element: ocgis_lh(exc=DefinitionValidationError(self, 'Geometry dictionaries must have a "geom" key.')) if 'properties' not in element: element['properties'] = {self._ugid_key: ii} crs = element.get('crs', CFWGS84()) if 'crs' not in element: ocgis_lh(msg='No CRS in geometry dictionary - assuming WGS84.', level=logging.WARN) ret = SpatialDimension.from_records(value, crs=crs, uid=self.geom_uid) else: if len(value) == 2: geom = Point(value[0], value[1]) elif len(value) == 4: minx, miny, maxx, maxy = value geom = Polygon(((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny))) if not geom.is_valid: raise DefinitionValidationError(self, 'Parsed geometry is not valid.') ret = [{'geom': geom, 'properties': {self._ugid_key: 1}}] ret = SpatialDimension.from_records(ret, crs=CFWGS84(), uid=self.geom_uid) self._bounds = geom.bounds elif isinstance(value, GeomCabinetIterator): self._shp_key = value.key or value.path # always want to yield SpatialDimension objects value.as_spatial_dimension = True ret = value elif isinstance(value, BaseGeometry): ret = [{'geom': value, 'properties': {self._ugid_key: 1}}] ret = SpatialDimension.from_records(ret, crs=CFWGS84(), uid=self.geom_uid) elif value is None: ret = value elif isinstance(value, SpatialDimension): ret = value else: raise NotImplementedError(type(value)) # convert to a tuple if this is a SpatialDimension object if isinstance(ret, SpatialDimension): ret = tuple(self._iter_spatial_dimension_tuple_(ret)) return ret
def test_get_spatial_subset_output_crs(self): """Test subsetting with an output CRS.""" # test with default crs converting to north american lambert proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs' output_crs = CoordinateReferenceSystem(proj4=proj4) subset_sdim = SpatialDimension.from_records([self.nebraska]) rd = self.test_data.get_rd('cancm4_tas') ss = SpatialSubsetOperation(rd, output_crs=output_crs) ret = ss.get_spatial_subset('intersects', subset_sdim) self.assertEqual(ret.spatial.crs, output_crs) self.assertAlmostEqual(ret.spatial.grid.value.mean(), -35065.750850951554) # test with an input rotated pole coordinate system rd = self.rd_rotated_pole ss = SpatialSubsetOperation(rd, output_crs=env.DEFAULT_COORDSYS) subset_sdim = SpatialDimension.from_records([self.germany]) ret = ss.get_spatial_subset('intersects', subset_sdim) self.assertEqual(ret.spatial.crs, env.DEFAULT_COORDSYS)
def test_write(self): records = [{'geom': Point(1, 2).buffer(1), 'properties': {'ID': 5, 'name': 'heaven'}}, {'geom': Point(7, 8).buffer(1), 'properties': {'ID': 50, 'name': 'hell'}}] sdim1 = SpatialDimension.from_records([records[0]], uid='ID') sdim2 = SpatialDimension.from_records([records[1]], uid='ID') field = self.get_field(crs=WGS84()) coll1 = SpatialCollection() coll1.add_field(field, ugeom=sdim1) coll2 = SpatialCollection() coll2.add_field(field, ugeom=sdim2) colls = [coll1, coll2] f = FakeAbstractCollectionConverter(colls, outdir=self.current_dir_output, prefix='me') ret = f.write() path = os.path.join(ret, 'shp', 'me_ugid.shp') with fiona.open(path, 'r') as source: records = list(source) self.assertEqual(len(records), 2) self.assertEqual([r['properties']['ID'] for r in records], [5, 50]) self.assertEqual([r['properties']['name'] for r in records], ['heaven', 'hell'])
def test_get_spatial_subset_circular_geometries(self): """Test circular geometries. They were causing wrapping errors.""" geoms = TestGeom.get_geometry_dictionaries() rd = self.test_data.get_rd('cancm4_tas') ss = SpatialSubsetOperation(rd, wrap=True) buffered = [element['geom'].buffer(rd.get().spatial.grid.resolution*2) for element in geoms] for buff in buffered: record = [{'geom': buff, 'properties': {'UGID': 1}}] subset_sdim = SpatialDimension.from_records(record) ret = ss.get_spatial_subset('intersects', subset_sdim) self.assertTrue(np.all(ret.spatial.grid.extent > 0))
def get_subset_sdim(self): # 1: nebraska nebraska = self.nebraska # 2: germany germany = self.germany # 3: nebraska and germany ret = [SpatialDimension.from_records(d) for d in [[nebraska], [germany], [nebraska, germany]]] return ret
def test_get_spatial_subset_wrap(self): """Test subsetting with wrap set to a boolean value.""" subset_sdim = SpatialDimension.from_records([self.nebraska]) rd = self.test_data.get_rd('cancm4_tas') self.assertEqual(rd.get().spatial.wrapped_state, WrappableCoordinateReferenceSystem._flag_unwrapped) ss = SpatialSubsetOperation(rd, wrap=True) ret = ss.get_spatial_subset('intersects', subset_sdim) self.assertEqual(ret.spatial.wrapped_state, WrappableCoordinateReferenceSystem._flag_wrapped) self.assertAlmostEqual(ret.spatial.grid.value.data[1].mean(), -99.84375) # test with wrap false ss = SpatialSubsetOperation(rd, wrap=False) ret = ss.get_spatial_subset('intersects', subset_sdim) self.assertEqual(ret.spatial.wrapped_state, WrappableCoordinateReferenceSystem._flag_unwrapped) self.assertAlmostEqual(ret.spatial.grid.value.data[1].mean(), 260.15625)
def test_prepare_subset_sdim(self): for subset_sdim in self.get_subset_sdim(): for ss, k in self: try: prepared = ss._prepare_subset_sdim_(subset_sdim) # check that a deepcopy has occurred self.assertFalse(np.may_share_memory(prepared.uid, subset_sdim.uid)) except KeyError: # the target has a rotated pole coordinate system. transformations to rotated pole for the subset # geometry is not supported. if isinstance(ss.sdim.crs, CFRotatedPole): continue else: raise self.assertEqual(prepared.crs, ss.sdim.crs) # test nebraska against an unwrapped dataset specifically nebraska = SpatialDimension.from_records([self.nebraska]) field = self.test_data.get_rd('cancm4_tas').get() ss = SpatialSubsetOperation(field) prepared = ss._prepare_subset_sdim_(nebraska) self.assertEqual(prepared.wrapped_state, WrappableCoordinateReferenceSystem._flag_unwrapped)
def test_write_fiona(self): keywords = dict(with_realization=[True, False], with_level=[True, False], with_temporal=[True, False], driver=['ESRI Shapefile', 'GeoJSON'], melted=[False, True]) for ii, k in enumerate(self.iter_product_keywords(keywords)): path = os.path.join(self.current_dir_output, '{0}'.format(ii)) field = self.get_field(with_value=True, crs=WGS84(), with_dimension_names=False, with_realization=k.with_realization, with_level=k.with_level, with_temporal=k.with_temporal) newvar = deepcopy(field.variables.first()) newvar.alias = 'newvar' newvar.value += 10 field.variables.add_variable(newvar, assign_new_uid=True) field = field[:, 0:2, :, 0:2, 0:2] field.write_fiona(path, driver=k.driver, melted=k.melted) with fiona.open(path) as source: records = list(source) if k.melted: dd = {a: [] for a in field.variables.keys()} for r in records: dd[r['properties']['alias']].append(r['properties']['value']) for kk, v in dd.iteritems(): self.assertAlmostEqual(np.mean(v), field.variables[kk].value.mean(), places=6) else: for alias in field.variables.keys(): values = [r['properties'][alias] for r in records] self.assertAlmostEqual(np.mean(values), field.variables[alias].value.mean(), places=6) n = reduce(lambda x, y: x * y, field.shape) if k.melted: n *= len(field.variables) self.assertEqual(n, len(records)) # test with a point abstraction field = self.get_field(with_value=True, crs=WGS84()) field = field[0, 0, 0, 0, 0] field.spatial.abstraction = 'point' path = self.get_temporary_file_path('foo.shp') field.write_fiona(path) with fiona.open(path) as source: gtype = source.meta['schema']['geometry'] self.assertEqual(gtype, 'Point') # test with a fake object passed in as a fiona object. this should raise an exception as the method will attempt # to use the object instead of creating a new collection. the object should not be closed when done. class DontHateMe(Exception): pass class WriteMe(Exception): pass class Nothing(object): def close(self): raise DontHateMe() def write(self, *args, **kwargs): raise WriteMe() with self.assertRaises(WriteMe): field.write_fiona(path, fobject=Nothing()) # Test all geometries are accounted for as well as properties. path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp') rd = RequestDataset(path) field = rd.get() out = self.get_temporary_file_path('foo.shp') field.write_fiona(out) with fiona.open(out, 'r') as source: for record in source: target = shape(record['geometry']) self.assertEqual(record['properties'].keys(), [u'UGID', u'STATE_FIPS', u'ID', u'STATE_NAME', u'STATE_ABBR']) found = False for geom in field.spatial.abstraction_geometry.value.flat: if target.almost_equals(geom): found = True break self.assertTrue(found) # Test with upper keys. field = self.get_field(with_value=True, crs=WGS84())[0, 0, 0, 0, 0] path = self.get_temporary_file_path('what.shp') field.write_fiona(path=path, use_upper_keys=True) with fiona.open(path) as source: for row in source: for key in row['properties']: self.assertTrue(key.isupper()) # test with upper keys field = self.get_field(with_value=True, crs=WGS84())[0, 0, 0, 0, 0] path = self.get_temporary_file_path('what2.shp') headers = ['time', 'tid'] field.write_fiona(path=path, headers=headers) with fiona.open(path) as source: self.assertEqual(source.meta['schema']['properties'].keys(), headers) # test passing a ugid field = self.get_field(with_value=True, crs=WGS84())[0, 0, 0, 0, 0] path = self.get_temporary_file_path('what3.shp') record = {'geom': Point(1, 2), 'properties': {'ugid': 10}} ugeom = SpatialDimension.from_records([record], uid='ugid') field.write_fiona(path=path, ugeom=ugeom) with fiona.open(path) as source: for row in source: self.assertEqual(row['properties'][constants.HEADERS.ID_SELECTION_GEOMETRY], 10)
def test_get_iter(self): field = self.get_field(with_value=True) rows = list(field.get_iter()) self.assertEqual(len(rows), 2 * 31 * 2 * 3 * 4) self.assertEqual(len(rows[0]), 2) self.assertEqual(rows[100][0].bounds, (-100.5, 38.5, -99.5, 39.5)) real = {'vid': 1, 'ub_time': datetime.datetime(2000, 1, 6, 0, 0), 'year': 2000, 'gid': 5, 'ub_level': 100, 'rid': 1, 'realization': 1, 'lb_level': 0, 'variable': 'tmax', 'month': 1, 'lb_time': datetime.datetime(2000, 1, 5, 0, 0), 'day': 5, 'level': 50, 'did': None, 'value': 0.32664490177209615, 'alias': 'tmax', 'lid': 1, 'time': datetime.datetime(2000, 1, 5, 12, 0), 'tid': 5, 'name': 'tmax', 'ugid': 1} self.assertAsSetEqual(rows[100][1].keys(), real.keys()) for k, v in rows[100][1].iteritems(): self.assertEqual(real[k], v) self.assertEqual(set(field.variables['tmax'].value.flatten().tolist()), set([r[1]['value'] for r in rows])) # Test without names. field = self.get_field(with_value=True, with_dimension_names=False) rows = list(field.get_iter()) self.assertAsSetEqual(rows[10][1].keys(), ['lid', 'name', 'vid', 'ub_time', 'did', 'lb_level', 'time', 'year', 'value', 'month', 'alias', 'tid', 'ub_level', 'rlz', 'variable', 'gid', 'rid', 'level', 'lb_time', 'day', 'ugid']) # Test not melted. field = self.get_field(with_value=True) other_variable = deepcopy(field.variables.first()) other_variable.alias = 'two' other_variable.value *= 2 field.variables.add_variable(other_variable, assign_new_uid=True) rows = list(field.get_iter(melted=False)) self.assertEqual(len(rows), 1488) for row in rows: attrs = row[1] # Test variable aliases are in the row dictionaries. for variable in field.variables.itervalues(): self.assertIn(variable.alias, attrs) # Test for upper keys. field = self.get_field(with_value=True)[0, 0, 0, 0, 0] for row in field.get_iter(use_upper_keys=True): for key in row[1].keys(): self.assertTrue(key.isupper()) # Test passing limiting headers. field = self.get_field(with_value=True) headers = ['time', 'tid'] for _, row in field.get_iter(headers=headers): self.assertEqual(row.keys(), headers) # Test passing a selection geometry identifier. field = self.get_field(with_value=True)[0, 0, 0, 0, 0] record = {'geom': Point(1, 2), 'properties': {'HI': 50, 'goodbye': 'forever'}} ugeom = SpatialDimension.from_records([record], uid='HI') _, row = field.get_iter(ugeom=ugeom).next() self.assertEqual(row['HI'], 50) # Test value keys. field = self.get_field(with_value=True)[0, 0, 0, 0, 0] fill = np.ma.array(np.zeros(2, dtype=[('a', float), ('b', float)])) value = np.ma.array(np.zeros(field.shape, dtype=object), mask=False) value.data[0, 0, 0, 0, 0] = fill field.variables['tmax']._value = value value_keys = ['a', 'b'] _, row = field.get_iter(value_keys=value_keys, melted=True) for vk in value_keys: self.assertIn(vk, row[1])