def test_overlay(dfs, how, use_sindex, expected_features): """ Basic overlay test with small dummy example dataframes (from docs). Results obtained using QGIS 2.16 (Vector -> Geoprocessing Tools -> Intersection / Union / ...), saved to GeoJSON and pasted here """ df1, df2 = dfs result = overlay(df1, df2, how=how, use_sindex=use_sindex) # construction of result if how == 'identity': expected = pd.concat([ GeoDataFrame.from_features(expected_features['intersection']), GeoDataFrame.from_features(expected_features['difference']) ], ignore_index=True) else: expected = GeoDataFrame.from_features(expected_features[how]) # TODO needed adaptations to result # if how == 'union': # result = result.drop(['idx1', 'idx2'], axis=1).sort_values(['col1', 'col2']).reset_index(drop=True) # elif how in ('intersection', 'identity'): # result = result.drop(['idx1', 'idx2'], axis=1) assert_geodataframe_equal(result, expected) # for difference also reversed if how == 'difference': result = overlay(df2, df1, how=how, use_sindex=use_sindex) expected = GeoDataFrame.from_features( expected_features['difference_inverse']) assert_geodataframe_equal(result, expected)
def test_from_features_unaligned_properties(self): p1 = Point(1, 1) f1 = { "type": "Feature", "properties": {"a": 0}, "geometry": p1.__geo_interface__, } p2 = Point(2, 2) f2 = { "type": "Feature", "properties": {"b": 1}, "geometry": p2.__geo_interface__, } p3 = Point(3, 3) f3 = { "type": "Feature", "properties": {"a": 2}, "geometry": p3.__geo_interface__, } df = GeoDataFrame.from_features([f1, f2, f3]) result = df[["a", "b"]] expected = pd.DataFrame.from_dict( [{"a": 0, "b": np.nan}, {"a": np.nan, "b": 1}, {"a": 2, "b": np.nan}] ) assert_frame_equal(expected, result)
def to_gdf( self, columns: list = None ) -> Union[GeoDataFrame, Dict[str, GeoDataFrame]]: """ Export an ArcGIS Server layer to GeoDataFrame :param columns: list of column names, optional Optionally specify the column names to include in the output frame. This does not overwrite the property names of the input, but can ensure a consistent output format. :return: """ gdf = GeoDataFrame.from_features(features=(self._features(layer) for layer in self.layer), crs=self.crs, columns=columns) for field in self.meta["fields"]: if field["type"] == "esriFieldTypeOID": gdf.set_index(field["name"], inplace=True) if field["type"] == "esriFieldTypeDate": gdf[field["name"]] = to_datetime(gdf[field["name"]], unit="ms") if field["type"] == "esriFieldTypeInteger": gdf[field["name"]] = gdf[field["name"]].astype("Int64") return gdf
def load_data_into_KDtree(path_to_dataset): """ Purpose: Reads geospatial data from a file and store it in a KD-tree, numpy array, and geopandas dataframe Inputs: `path_to_dataset`: A string of the path to the 'dataset.geojson' file Outputs: `datasetKD`: A KD-tree of the .geojson data `datasetDF`: A Geopandas dataframe of the .geojson data `datasetNP': A numpy array of the .geojson data points """ data = '' if path.isfile(path_to_dataset): with open(path_to_dataset, 'r') as f: data = f.read() else: print( "Incorrect path to dataset. Check DATASET_MAP and see if the path is correct." ) dataset = loads(data) # creates a geopandas dataframe with the data datasetDF = GeoDataFrame.from_features(dataset, crs="EPSG:4326") # since cKDTree needs an array-like object to query nearest neighbors, we create # such an array datasetNP = array(list(datasetDF.geometry.apply(lambda x: (x.x, x.y)))) # populate the KD tree datasetKD = cKDTree(datasetNP) return (datasetKD, datasetDF, datasetNP)
def make_geojson_links( ref_layer_geojson, csv_table, field_i, field_j, field_fij, join_field): gdf = GeoDataFrame.from_features(ref_layer_geojson["features"]) gdf.loc[:, join_field] = gdf.loc[:, join_field].astype(str) gdf.set_index(join_field, inplace=True, drop=False) gdf.geometry = _compute_centroids(gdf.geometry) table = pd_read_json(csv_table) table.loc[:, (field_i, field_j)] = \ table.loc[:, (field_i, field_j)].astype(str) table = \ table[table[field_i].isin(gdf.index) & table[field_j].isin(gdf.index)] geoms_loc = gdf.geometry.loc ft_template_start = \ '''{"type":"Feature","geometry":{"type":"LineString","coordinates":[''' geojson_features = [] for n, id_i, id_j, fij in table[[field_i, field_j, field_fij]].itertuples(): pts = \ list(geoms_loc[id_i].coords)[0] + list(geoms_loc[id_j].coords)[0] geojson_features.append(''.join([ ft_template_start, '''[{0},{1}],[{2},{3}]'''.format(*pts), ''']},"properties":{"''', '''{0}":"{1}","{2}":"{3}","{4}":"{5}"''' .format(field_i, id_i, field_j, id_j, field_fij, fij), '''}}''' ]) ) return ''.join([ '''{"type":"FeatureCollection","features":[''', ','.join(geojson_features), ''']}''' ]).encode()
def make_geojson_links(ref_layer_geojson, csv_table, field_i, field_j, field_fij, join_field): gdf = GeoDataFrame.from_features(ref_layer_geojson["features"]) gdf.set_index(join_field, inplace=True, drop=False) gdf.geometry = _compute_centroids(gdf.geometry) csv_table = pd_read_json(csv_table) csv_table = csv_table[csv_table["i"].isin(gdf.index) & csv_table["j"].isin(gdf.index)] geoms_loc = gdf.geometry.loc ft_template_start = \ '''{"type":"Feature","geometry":{"type":"LineString","coordinates":[''' geojson_features = [] for n, id_i, id_j, fij in csv_table[[field_i, field_j, field_fij]].itertuples(): # pt1, pt2 = \ # list(geoms_loc[id_i].coords)[0], list(geoms_loc[id_j].coords)[0] pts = \ list(geoms_loc[id_i].coords)[0] + list(geoms_loc[id_j].coords)[0] geojson_features.append(''.join([ ft_template_start, '''[{0},{1}],[{2},{3}]'''.format(*pts), ''']},"properties":{"''', '''i":"{0}","j":"{1}","fij":"{2}"'''.format(id_i, id_j, fij), '''}}''' ])) return ''.join([ '''{"type":"FeatureCollection","crs":{"type":"name","properties":''' '''{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"}},"features":[''', ','.join(geojson_features), ''']}''' ]).encode()
def convexQuery(): """ Purpose: Return the smallest convex polygon that contains a set of points Input: `BBparams`: a dictionary of the datasets to be queried, and the top left and bottom right corners of the bounding box Output: `convex_full_results['features']+intersections`: a list of the feature polygon of the convex hull and the points contained by it """ convexFC = {'type': "FeatureCollection", 'features': []} BBparams = loads(request.args.get("BBparams", None)) # store a feature list of points contained by the bounding box intersections, _ = queryBBoxIntersection(BBparams) # append to `convexFC` the polygon formed by all the points within the bbox convexFC['features'] = [createPolygonFromPoints(intersections)] # create a dataframe with the polygon convexParams = GeoDataFrame.from_features(convexFC, crs="EPSG:4326") # create a geoseries of points that form the smallest convex Polygon # "The convex hull of a geometry is the smallest convex Polygon containing # all the points in each geometry, unless the number of points in the geometric # object is less than three. For two points, the convex hull collapses to a # LineString; for 1, a Point. # - https://geopandas.readthedocs.io/en/latest/docs/reference/api/geopandas.GeoSeries.convex_hull.html convex_full_results = loads((convexParams.convex_hull).to_json()) # return a list of the convex hull and the intersecting points found within the bounding box return jsonify(convex_full_results['features'] + intersections)
def generate_decision_layer_cell_seabed(session, grid): grid_cells = session.query(models.DecisionLayerCell, models.GridCell).join( models.GridCell).filter(models.GridCell.grid_id == grid.id).order_by( models.GridCell.id).all() for cell in grid_cells: grid_box = to_shape(cell.GridCell.bounding_box) lonmin, lonmax, latmin, latmax = grid_box.bounds bbox_string = ",".join(map(str, [latmin, lonmin, latmax, lonmax])) wfs_url = "http://drive.emodnet-geology.eu/geoserver/EMODnetGeology/ows?service=WFS&version=1.1.0&request=GetFeature&typeName=EMODnetGeology:seabed_substrate250k_eu_sbss250k_seabed_substrate_eu&bbox=" + bbox_string + "&outputFormat=application%2Fjson" try: r = requests.get(wfs_url) wfs_geo = geojson.loads(r.content) wfs_gdf = GeoDataFrame.from_features(wfs_geo) if len(wfs_gdf) > 0: if len(wfs_gdf[wfs_gdf.folk_5_substrate_class.str.contains( '2. Sand')]) > 0: cell.DecisionLayerCell.seabed = wfs_gdf[ wfs_gdf.folk_5_substrate_class.str.contains( '2. Sand')].shape_area.sum() / grid_box.area * 100 else: cell.DecisionLayerCell.seabed = 0 except ValueError: pass session.commit()
def calc_zonal_statistics(shp_filename, raster_file, stats=None): """ Calculates zonal statistics for a raster file based on a shapefile. :param shp_filename: filename path. :type shp_filename: str :param raster_file: filename raster. :type raster_file: str :param stats: list of statistics to calculate. :type stats: list :return: gpd.GeoDataFrame """ if stats is None: stats = ['count', 'min', 'max', 'mean', 'median', 'std'] results = zonal_stats( shp_filename, raster_file, nodata=-9999, stats=stats, all_touched=True, geojson_out=True, epsg=4326 ) gdf_stats = GeoDataFrame.from_features(results) return gdf_stats
def make_geojson_links(ref_layer_geojson, csv_table, field_i, field_j, field_fij, join_field): gdf = GeoDataFrame.from_features(ref_layer_geojson["features"]) gdf.loc[:, join_field] = gdf.loc[:, join_field].astype(str) gdf.set_index(join_field, inplace=True, drop=False) gdf.geometry = _compute_centroids(gdf.geometry) table = pd_read_json(csv_table) table.loc[:, (field_i, field_j)] = \ table.loc[:, (field_i, field_j)].astype(str) table = \ table[table[field_i].isin(gdf.index) & table[field_j].isin(gdf.index)] geoms_loc = gdf.geometry.loc ft_template_start = \ '''{"type":"Feature","geometry":{"type":"LineString","coordinates":[''' geojson_features = [] for n, id_i, id_j, fij in table[[field_i, field_j, field_fij]].itertuples(): pts = \ list(geoms_loc[id_i].coords)[0] + list(geoms_loc[id_j].coords)[0] geojson_features.append(''.join([ ft_template_start, '''[{0},{1}],[{2},{3}]'''.format(*pts), ''']},"properties":{"''', '''{0}":"{1}","{2}":"{3}","{4}":"{5}"'''.format( field_i, id_i, field_j, id_j, field_fij, fij), '''}}''' ])) return ''.join([ '''{"type":"FeatureCollection","features":[''', ','.join(geojson_features), ''']}''' ]).encode()
def get_example_aoi( self, location: str = "Berlin", as_dataframe: bool = False) -> Union[dict, GeoDataFrame]: """ Gets predefined, small, rectangular example aoi for the selected location. Args: location: Location, one of Berlin, Washington. as_dataframe: Returns a dataframe instead of dict FeatureColletions (default). Returns: Feature collection json with the selected aoi. """ logger.info(f"Getting small example aoi in location '{location}'.") if location == "Berlin": example_aoi = self.read_vector_file( f"{str(Path(__file__).resolve().parent)}/data/aoi_berlin.geojson" ) elif location == "Washington": example_aoi = self.read_vector_file( f"{str(Path(__file__).resolve().parent)}/data/aoi_washington.geojson" ) else: raise ValueError( "Please select one of 'Berlin' or 'Washington' as the location!" ) if as_dataframe: df = GeoDataFrame.from_features(example_aoi, crs=4326) return df else: return example_aoi
def read_file(filename, bbox=None, **kwargs): """ Returns a GeoDataFrame from a file or URL. Parameters ---------- filename: str Either the absolute or relative path to the file or URL to be opened. bbox : tuple | GeoDataFrame or GeoSeries, default None Filter features by given bounding box, GeoSeries, or GeoDataFrame. CRS mis-matches are resolved if given a GeoSeries or GeoDataFrame. **kwargs: Keyword args to be passed to the `open` or `BytesCollection` method in the fiona library when opening the file. For more information on possible keywords, type: ``import fiona; help(fiona.open)`` Examples -------- >>> df = geopandas.read_file("nybb.shp") Returns ------- geodataframe : GeoDataFrame """ if _is_url(filename): req = _urlopen(filename) path_or_bytes = req.read() reader = fiona.BytesCollection else: path_or_bytes = filename reader = fiona.open with fiona_env(): with reader(path_or_bytes, **kwargs) as features: # In a future Fiona release the crs attribute of features will # no longer be a dict. The following code will be both forward # and backward compatible. if hasattr(features.crs, "to_dict"): crs = features.crs.to_dict() else: crs = features.crs if bbox is not None: if isinstance(bbox, GeoDataFrame) or isinstance( bbox, GeoSeries): bbox = tuple(bbox.to_crs(crs).total_bounds) assert len(bbox) == 4 f_filt = features.filter(bbox=bbox) else: f_filt = features columns = list( features.meta["schema"]["properties"]) + ["geometry"] gdf = GeoDataFrame.from_features(f_filt, crs=crs, columns=columns) return gdf
def compute(self): self.output.create_output_dir(self.output.uri) features = gdal_zonalstats( self.inputs[1].read(format=formats.JSON, epsg=self.inputs[0].get_epsg()), self.inputs[0].read()) self.output.data = GeoDataFrame.from_features(features) self.output.write()
def geojson_2_geodataframe_features(geo): with BytesCollection(bytes(geojson.dumps(geo), encoding='utf8')) as features: crs = features.crs columns = list(features.meta["schema"]["properties"]) + ["geometry"] gdf = GeoDataFrame.from_features(features, crs=crs) gdf = gdf[columns] return gdf
def load_footprints(footprint_file_name: str) -> GeoDataFrame: with fiona.open(footprint_file_name) as features: properties_to_keep = [] # None of the properties here are useful df = GeoDataFrame.from_features( features_slimmed(features, properties_to_keep)) df.crs = features.crs return remove_invalid_geometries(df.to_crs(epsg=4326))
def vectorize(src=None, image=None, transform=None, crs=None): """ Raster-to-Vector conversion. Performs a raster-to-vector conversion of a classified image. Parameters ---------- src: Rasterio datasource A rasterio-style datasource created using: with rasterio.open('path') as src. The datasource referred to must be a classified image. This parameter is optional. If it is not provided then the image and the transform must be provided. image: numpy.array A signle band of (classified, ideally) image data where the pixel values are integers. Shape is (1, rows, columns). This parameter is optional. transform: rasterio.transform A raster transform used to convert row/column values to geographic coordinates. This parameter is optional. crs: rasterio.crs A proj4 string representing the coordinate reference system. This parameter is optional. Returns ------- GeoDataFrame A vector version of the classified raster. """ if src is not None: img = src.read(1, masked=True) transform = src.transform crs = src.crs.to_proj4() else: img = image[0].astype(np.int32) # shps = features.shapes(img, connectivity=8, transform=transform) shps = features.shapes(img, transform=transform) records = [] for id, shp in enumerate(shps): if shp[1] != 0: item = { 'geometry': shp[0], 'id': id + 1, 'properties': OrderedDict([('dn', np.int(shp[1]))]), 'type': 'Feature' } records.append(item) vec = GeoDataFrame.from_features(records) vec.crs = crs return vec
def json_obj_to_geodf(json_obj, epsg): """ Json Object to GeoDataFrame """ from geopandas import GeoDataFrame return GeoDataFrame.from_features(json_obj['features'], 'EPSG:{}'.format(str(epsg)))
def test_from_features(self): nybb_filename = geopandas.datasets.get_path('nybb') with fiona.open(nybb_filename) as f: features = list(f) crs = f.crs df = GeoDataFrame.from_features(features, crs=crs) validate_boro_df(df, case_sensitive=True) assert df.crs == crs
def json_obj_to_geodf(json_obj, epsg): """ Json Object to GeoDataFrame """ from geopandas import GeoDataFrame return GeoDataFrame.from_features(json_obj['features'], {'init': 'epsg:{}'.format(epsg)})
def read_file(filename, bbox=None, **kwargs): """ Returns a GeoDataFrame from a file or URL. Parameters ---------- filename: str Either the absolute or relative path to the file or URL to be opened. bbox : tuple | GeoDataFrame or GeoSeries, default None Filter features by given bounding box, GeoSeries, or GeoDataFrame. CRS mis-matches are resolved if given a GeoSeries or GeoDataFrame. **kwargs: Keyword args to be passed to the `open` or `BytesCollection` method in the fiona library when opening the file. For more information on possible keywords, type: ``import fiona; help(fiona.open)`` Examples -------- >>> df = geopandas.read_file("nybb.shp") Returns ------- geodataframe : GeoDataFrame """ if _is_url(filename): req = _urlopen(filename) path_or_bytes = req.read() reader = fiona.BytesCollection else: path_or_bytes = filename reader = fiona.open with fiona_env(): with reader(path_or_bytes, **kwargs) as features: # In a future Fiona release the crs attribute of features will # no longer be a dict. The following code will be both forward # and backward compatible. if hasattr(features.crs, 'to_dict'): crs = features.crs.to_dict() else: crs = features.crs if bbox is not None: if isinstance(bbox, GeoDataFrame) or isinstance(bbox, GeoSeries): bbox = tuple(bbox.to_crs(crs).total_bounds) assert len(bbox) == 4 f_filt = features.filter(bbox=bbox) else: f_filt = features columns = list(features.meta["schema"]["properties"]) + ["geometry"] gdf = GeoDataFrame.from_features(f_filt, crs=crs, columns=columns) return gdf
def add_message(): data = request.json gdf_points: GeoDataFrame = GeoDataFrame.from_features(data["features"]) polygon: Polygon = gdf_points['geometry'].agg(lambda point: Polygon(point.tolist())) gdf_poly = GeoDataFrame([{'id': 1, 'geometry': polygon}]) print(gdf_poly.to_json()) return gdf_poly.to_json()
def test_from_features(self): nybb_filename = geopandas.datasets.get_path("nybb") with fiona.open(nybb_filename) as f: features = list(f) crs = f.crs df = GeoDataFrame.from_features(features, crs=crs) validate_boro_df(df, case_sensitive=True) assert df.crs == crs
def search(self, search_parameters: dict, as_dataframe: bool = True) -> Union[GeoDataFrame, dict]: """ Searches the catalog for the the search parameters and returns the metadata of the matching scenes. Args: search_parameters: The catalog search parameters, see example. as_dataframe: return type, GeoDataFrame if True (default), FeatureCollection if False. Returns: The search results as a GeoDataFrame, optionally as json dict. Example: ```python search_parameters={ "datetime": "2019-01-01T00:00:00Z/2019-01-15T23:59:59Z", "intersects": { "type": "Polygon", "coordinates": [[[13.32113746,52.73971768],[13.15981158,52.2092959], [13.62204483,52.15632025],[13.78859517,52.68655119],[13.32113746, 52.73971768]]]}, "limit": 10, "sortby": [{"field" : "properties.acquisitionDate", "direction" : "asc"}] } ``` """ logger.info( f"Searching catalog with search_parameters: {search_parameters}") url = f"{self.auth._endpoint()}/catalog/stac/search" response_json: dict = self.auth._request("POST", url, search_parameters) logger.info(f"{len(response_json['features'])} results returned.") dst_crs = "EPSG:4326" df = GeoDataFrame.from_features(response_json, crs=dst_crs) if df.empty: if as_dataframe: return df else: return df.__geo_interface__ # Filter to actual geometries intersecting the aoi (Sobloo search uses a rectangular # bounds geometry, can contain scenes that touch the aoi bbox, but not the aoi. # So number returned images not consistent with set limit. # TODO: Resolve on backend geometry = search_parameters["intersects"] poly = shape(geometry) df = df[df.intersects(poly)] df = df.reset_index(drop=True) df.crs = dst_crs # apply resets the crs if as_dataframe: return df else: return df.__geo_interface__
def test_from_features(self): nybb_filename, nybb_zip_path = download_nybb() with fiona.open(nybb_zip_path, vfs='zip://' + nybb_filename) as f: features = list(f) crs = f.crs df = GeoDataFrame.from_features(features, crs=crs) df.rename(columns=lambda x: x.lower(), inplace=True) validate_boro_df(self, df) self.assert_(df.crs == crs)
def test_from_features(self): nybb_filename = download_nybb() with fiona.open("/nybb_14a_av/nybb.shp", vfs="zip://" + nybb_filename) as f: features = list(f) crs = f.crs df = GeoDataFrame.from_features(features, crs=crs) df.rename(columns=lambda x: x.lower(), inplace=True) validate_boro_df(self, df) self.assert_(df.crs == crs)
def test_from_features(self): nybb_filename = geopandas.datasets.get_path('nybb') with fiona.open(nybb_filename) as f: features = list(f) crs = f.crs df = GeoDataFrame.from_features(features, crs=crs) df.rename(columns=lambda x: x.lower(), inplace=True) validate_boro_df(self, df) self.assert_(df.crs == crs)
def test_from_features(self): nybb_filename = geopandas.datasets.get_path('nybb') with fiona.open(nybb_filename) as f: features = list(f) crs = f.crs df = GeoDataFrame.from_features(features, crs=crs) df.rename(columns=lambda x: x.lower(), inplace=True) validate_boro_df(df) assert df.crs == crs
def network_as_dataframe(self): try: from geopandas import GeoDataFrame result = GeoDataFrame.from_features(self['features']) result['id'] = [f['id'] for f in self['features']] return result except Exception as e: print('Could not create GeoDataFrame. Using regular DataFrame.') print(str(e)) return self['features'].as_dataframe()
def load_parcels(parcel_file_name: str) -> GeoDataFrame: with fiona.open(parcel_file_name) as features: properties_to_keep = ['PARCELID', 'RESYRBLT'] slim_features = features_slimmed( features, properties_to_keep) # Reduces memory usage slim_features = ( f for f in slim_features if f['geometry'] is not None ) # Apparently there are things in here with no shape? gdf = GeoDataFrame.from_features(slim_features) gdf.crs = features.crs return remove_invalid_geometries(gdf.to_crs(epsg=4326))
def load_osu_buildings(building_file_name: str, data_dir: str) -> GeoDataFrame: # TODO find a way to download this file with fiona.open(building_file_name) as features: gdf = GeoDataFrame.from_features(features) gdf.crs = features.crs building_ages = download_osu_buildings_ages(building_file_name, data_dir) gdf = gdf.merge(building_ages, on='BLDG_NUM') gdf = gdf[['geometry', 'year_built']] # Remove all the columns we don't need return remove_invalid_geometries(gdf.to_crs(epsg=4326))
def test_from_feature_collection(self): data = {'name': ['a', 'b', 'c'], 'lat': [45, 46, 47.5], 'lon': [-120, -121.2, -122.9]} df = pd.DataFrame(data) geometry = [Point(xy) for xy in zip(df['lon'], df['lat'])] gdf = GeoDataFrame(df, geometry=geometry) # from_features returns sorted columns expected = gdf[['geometry', 'lat', 'lon', 'name']] # test FeatureCollection res = GeoDataFrame.from_features(gdf.__geo_interface__) assert_frame_equal(res, expected) # test list of Features res = GeoDataFrame.from_features(gdf.__geo_interface__['features']) assert_frame_equal(res, expected) # test __geo_interface__ attribute (a GeoDataFrame has one) res = GeoDataFrame.from_features(gdf) assert_frame_equal(res, expected)
def queryBBoxIntersection(BBparams): """ Purpose: Performs a bounding box query on all selected datasets Input: `BBparams`: a dictionary of the datasets to be queried, and the top left and bottom right corners of the bounding box Output: `full_results`: a list of the feature documents of all points contained within the bounding box `bbox['features']`: a feature document of the bounding box """ # store the top, left, bottom, and right coordinate borders of the bounding box top, left, bottom, right = \ float(BBparams["bbox"][0].split(',')[0]), \ float(BBparams["bbox"][0].split(',')[1]), \ float(BBparams["bbox"][1].split(',')[0]), \ float(BBparams["bbox"][1].split(',')[1]) # the bounding box's polygon feature bbox = { 'type': "FeatureCollection", "features": [{ "type": "Feature", "properties": { "name": "United States" }, "geometry": { "type": "Polygon", "coordinates": [[[top, left], [top, right], [bottom, right], [bottom, left], [top, left]]] } }] } # create a dataframe of the `bbox` feature bbox_df = GeoDataFrame.from_features(bbox, crs="EPSG:4326") # load the datasets into memory process_datasets(BBparams["datasets"]) full_results = [] for dataset in dataset_map: # only query the datasets that were checkboxed in the frontend if BBparams['datasets'][dataset]: # queries the dataset dataframe for all points in `bbox_df`'s bounding box points_in_bbox, _ = bbox_df.sindex.query_bulk( dataset_map[dataset]["dataframe"].geometry, predicate='intersects') # create a dataframe of all the intersecting points in dataset dataframe matches = dataset_map[dataset]["dataframe"].iloc[points_in_bbox] full_results += loads(matches.to_json())['features'] return full_results, bbox['features']
def dump_elbs(year=2016): storage_key = settings['azure']['pcs_storage_key'] account = az.CloudStorageAccount(account_name='pcslive', account_key=storage_key) blob_service = account.create_block_blob_service() year_ids = elb_repo.get_elb_harvest_year_ids(year=2016) if not os.path.exists('data/elbs'): os.mkdir('data/elbs') for idx, elb_year_id in enumerate(year_ids): print("downloading elb GIS cells. idx, yearid: ({} of {}), {}".format(idx, len(year_ids), elb_year_id)) crop = gis_repo.get_pps_crop(elb_year_id) if not 'Corn' in crop: print("found not-corn crop, ignoring: {}".format(crop)) continue # use the harvest layers elb_source_layers = [ b.name for b in list(blob_service.list_blobs('sourcelayers', str(elb_year_id))) if any(x in b.name for x in ['_13_', '_14_', '_15_'])] elb_harvest_source_layer_name = elb_source_layers[0] if len(elb_source_layers) > 0 else None if elb_harvest_source_layer_name is None: print("ELB has no harvest layer: {}".format(elb_year_id)) continue blob_zip = blob_service.get_blob_to_bytes('sourcelayers', elb_harvest_source_layer_name) vsiz = '/vsimem/{}.zip'.format(uuid.uuid4().hex) # gdal/ogr requires a .zip extension FileFromMemBuffer(vsiz, bytes(blob_zip.content)) with fiona.Collection(vsiz, vsi='zip') as f: shp = GeoDataFrame.from_features(f, crs={'init': 'epsg:4326'}) elb_points = GeoDataFrame(shp.loc[shp['ELB_ID'] > 0]) elb_centroids = list(elb_points.centroid) pps = gis_repo.processed_layer_shapes_by_year_id(elb_year_id) # get pps cells that have an elb pps_elb_cells = DataFrame( pps.loc[pps['geometry'].apply(lambda x: any(x.intersects(c) for c in elb_centroids))]) pps_elb_cells.drop(['geometry'], inplace=True, axis=1) # load weather record wx = gis_repo.weather_by_year_id(elb_year_id) pps_elb_cells = pandas.concat([ pps_elb_cells, pandas.DataFrame([wx.values], index=pps_elb_cells.index, columns=wx.keys())], axis=1) pps_elb_cells.to_pickle(f'data/elbs/{elb_year_id}_elb.pickle.gz', compression='gzip')
def from_dict(cls, data: dict) -> "FeatureCollection": """ Create a feature collection from a python dictionary that was created from the JSON definition of the FeatureCollection :param data: The dictionary that contains the feature collection definition :return: A new FeatureCollection object """ return cls( id=data["id"], data=GeoDataFrame.from_features(data["data"]), start_times=data.get("start_times"), end_times=data.get("end_times"), )
def test_fishnet_geojson_string_return(self): """ Tests fishnet generation with a GeoJSON string return """ self.logger.info('Fishnet with GeoJSON string return...') geojson = FishNet(outfile=None, outformat='GeoJSON', bbox=[414650, 563500, 429600, 575875]).create() self.assertFalse(geojson is None) try: gdf = GeoDataFrame.from_features(geojson) self.logger.info(gdf.head(10)) except ValueError: self.fail('Returned GeoJSON could not be read into a GeoDataFrame') self.logger.info('Completed')
def test_from_features_unaligned_properties(self): p1 = Point(1, 1) f1 = {"type": "Feature", "properties": {"a": 0}, "geometry": p1.__geo_interface__} p2 = Point(2, 2) f2 = {"type": "Feature", "properties": {"b": 1}, "geometry": p2.__geo_interface__} p3 = Point(3, 3) f3 = {"type": "Feature", "properties": {"a": 2}, "geometry": p3.__geo_interface__} df = GeoDataFrame.from_features([f1, f2, f3]) result = df[["a", "b"]] expected = pd.DataFrame.from_dict([{"a": 0, "b": np.nan}, {"a": np.nan, "b": 1}, {"a": 2, "b": np.nan}]) assert_frame_equal(expected, result)
def test_from_feature_collection(self): data = { "name": ["a", "b", "c"], "lat": [45, 46, 47.5], "lon": [-120, -121.2, -122.9], } df = pd.DataFrame(data) geometry = [Point(xy) for xy in zip(df["lon"], df["lat"])] gdf = GeoDataFrame(df, geometry=geometry) # from_features returns sorted columns expected = gdf[["geometry", "lat", "lon", "name"]] # test FeatureCollection res = GeoDataFrame.from_features(gdf.__geo_interface__) assert_frame_equal(res, expected) # test list of Features res = GeoDataFrame.from_features(gdf.__geo_interface__["features"]) assert_frame_equal(res, expected) # test __geo_interface__ attribute (a GeoDataFrame has one) res = GeoDataFrame.from_features(gdf) assert_frame_equal(res, expected)
def read_file(filename, **kwargs): """ Returns a GeoDataFrame from a file. *filename* is either the absolute or relative path to the file to be opened and *kwargs* are keyword args to be passed to the method when opening the file. """ bbox = kwargs.pop('bbox', None) with fiona.open(filename, **kwargs) as f: crs = f.crs if bbox is not None: assert len(bbox)==4 f_filt = f.filter(bbox=bbox) else: f_filt = f gdf = GeoDataFrame.from_features(f, crs=crs) return gdf
def read_file(filename, **kwargs): """ Returns a GeoDataFrame from a file. *filename* is either the absolute or relative path to the file to be opened and *kwargs* are keyword args to be passed to the `open` method in the fiona library when opening the file. For more information on possible keywords, type: ``import fiona; help(fiona.open)`` """ bbox = kwargs.pop('bbox', None) with fiona.open(filename, **kwargs) as f: crs = f.crs if bbox is not None: assert len(bbox)==4 f_filt = f.filter(bbox=bbox) else: f_filt = f gdf = GeoDataFrame.from_features(f_filt, crs=crs) return gdf
def test_from_features_unaligned_properties(self): p1 = Point(1,1) f1 = {'type': 'Feature', 'properties': {'a': 0}, 'geometry': p1.__geo_interface__} p2 = Point(2,2) f2 = {'type': 'Feature', 'properties': {'b': 1}, 'geometry': p2.__geo_interface__} p3 = Point(3,3) f3 = {'type': 'Feature', 'properties': {'a': 2}, 'geometry': p3.__geo_interface__} df = GeoDataFrame.from_features([f1, f2, f3]) result = df[['a', 'b']] expected = pd.DataFrame.from_dict([{'a': 0, 'b': np.nan}, {'a': np.nan, 'b': 1}, {'a': 2, 'b': np.nan}]) assert_frame_equal(expected, result)
def read_file(filename, **kwargs): """ Returns a GeoDataFrame from a file. *filename* is either the absolute or relative path to the file to be opened and *kwargs* are keyword args to be passed to the `open` method in the fiona library when opening the file. For more information on possible keywords, type: ``import fiona; help(fiona.open)`` """ bbox = kwargs.pop('bbox', None) with fiona.open(filename, **kwargs) as f: crs = f.crs if bbox is not None: assert len(bbox)==4 f_filt = f.filter(bbox=bbox) else: f_filt = f gdf = GeoDataFrame.from_features(f_filt, crs=crs) # re-order with column order from metadata, with geometry last columns = list(f.meta["schema"]["properties"]) + ["geometry"] gdf = gdf[columns] return gdf
def network_as_dataframe(self): from geopandas import GeoDataFrame result = GeoDataFrame.from_features(self['features']) result['id'] = [f['id'] for f in self['features']] return result