def test_overlay(dfs, how, use_sindex, expected_features):
"""
Basic overlay test with small dummy example dataframes (from docs).
Results obtained using QGIS 2.16 (Vector -> Geoprocessing Tools ->
Intersection / Union / ...), saved to GeoJSON and pasted here
"""
df1, df2 = dfs
result = overlay(df1, df2, how=how, use_sindex=use_sindex)
# construction of result
if how == 'identity':
expected = pd.concat([
GeoDataFrame.from_features(expected_features['intersection']),
GeoDataFrame.from_features(expected_features['difference'])
], ignore_index=True)
else:
expected = GeoDataFrame.from_features(expected_features[how])
# TODO needed adaptations to result
# if how == 'union':
# result = result.drop(['idx1', 'idx2'], axis=1).sort_values(['col1', 'col2']).reset_index(drop=True)
# elif how in ('intersection', 'identity'):
# result = result.drop(['idx1', 'idx2'], axis=1)
assert_geodataframe_equal(result, expected)
# for difference also reversed
if how == 'difference':
result = overlay(df2, df1, how=how, use_sindex=use_sindex)
expected = GeoDataFrame.from_features(
expected_features['difference_inverse'])
assert_geodataframe_equal(result, expected)
def test_from_features_unaligned_properties(self):
p1 = Point(1, 1)
f1 = {
"type": "Feature",
"properties": {"a": 0},
"geometry": p1.__geo_interface__,
}
p2 = Point(2, 2)
f2 = {
"type": "Feature",
"properties": {"b": 1},
"geometry": p2.__geo_interface__,
}
p3 = Point(3, 3)
f3 = {
"type": "Feature",
"properties": {"a": 2},
"geometry": p3.__geo_interface__,
}
df = GeoDataFrame.from_features([f1, f2, f3])
result = df[["a", "b"]]
expected = pd.DataFrame.from_dict(
[{"a": 0, "b": np.nan}, {"a": np.nan, "b": 1}, {"a": 2, "b": np.nan}]
)
assert_frame_equal(expected, result)
def to_gdf(
self,
columns: list = None
) -> Union[GeoDataFrame, Dict[str, GeoDataFrame]]:
"""
Export an ArcGIS Server layer to GeoDataFrame
:param columns: list of column names, optional
Optionally specify the column names to include in the output frame.
This does not overwrite the property names of the input, but can
ensure a consistent output format.
:return:
"""
gdf = GeoDataFrame.from_features(features=(self._features(layer)
for layer in self.layer),
crs=self.crs,
columns=columns)
for field in self.meta["fields"]:
if field["type"] == "esriFieldTypeOID":
gdf.set_index(field["name"], inplace=True)
if field["type"] == "esriFieldTypeDate":
gdf[field["name"]] = to_datetime(gdf[field["name"]], unit="ms")
if field["type"] == "esriFieldTypeInteger":
gdf[field["name"]] = gdf[field["name"]].astype("Int64")
return gdf
def load_data_into_KDtree(path_to_dataset):
"""
Purpose: Reads geospatial data from a file and store it in a KD-tree,
numpy array, and geopandas dataframe
Inputs: `path_to_dataset`: A string of the path to the 'dataset.geojson' file
Outputs: `datasetKD`: A KD-tree of the .geojson data
`datasetDF`: A Geopandas dataframe of the .geojson data
`datasetNP': A numpy array of the .geojson data points
"""
data = ''
if path.isfile(path_to_dataset):
with open(path_to_dataset, 'r') as f:
data = f.read()
else:
print(
"Incorrect path to dataset. Check DATASET_MAP and see if the path is correct."
)
dataset = loads(data)
# creates a geopandas dataframe with the data
datasetDF = GeoDataFrame.from_features(dataset, crs="EPSG:4326")
# since cKDTree needs an array-like object to query nearest neighbors, we create
# such an array
datasetNP = array(list(datasetDF.geometry.apply(lambda x: (x.x, x.y))))
# populate the KD tree
datasetKD = cKDTree(datasetNP)
return (datasetKD, datasetDF, datasetNP)
def make_geojson_links(
ref_layer_geojson, csv_table, field_i, field_j, field_fij, join_field):
gdf = GeoDataFrame.from_features(ref_layer_geojson["features"])
gdf.loc[:, join_field] = gdf.loc[:, join_field].astype(str)
gdf.set_index(join_field, inplace=True, drop=False)
gdf.geometry = _compute_centroids(gdf.geometry)
table = pd_read_json(csv_table)
table.loc[:, (field_i, field_j)] = \
table.loc[:, (field_i, field_j)].astype(str)
table = \
table[table[field_i].isin(gdf.index) & table[field_j].isin(gdf.index)]
geoms_loc = gdf.geometry.loc
ft_template_start = \
'''{"type":"Feature","geometry":{"type":"LineString","coordinates":['''
geojson_features = []
for n, id_i, id_j, fij in table[[field_i, field_j, field_fij]].itertuples():
pts = \
list(geoms_loc[id_i].coords)[0] + list(geoms_loc[id_j].coords)[0]
geojson_features.append(''.join([
ft_template_start,
'''[{0},{1}],[{2},{3}]'''.format(*pts),
''']},"properties":{"''',
'''{0}":"{1}","{2}":"{3}","{4}":"{5}"'''
.format(field_i, id_i, field_j, id_j, field_fij, fij),
'''}}'''
])
)
return ''.join([
'''{"type":"FeatureCollection","features":[''',
','.join(geojson_features),
''']}'''
]).encode()
def make_geojson_links(ref_layer_geojson, csv_table, field_i, field_j,
field_fij, join_field):
gdf = GeoDataFrame.from_features(ref_layer_geojson["features"])
gdf.set_index(join_field, inplace=True, drop=False)
gdf.geometry = _compute_centroids(gdf.geometry)
csv_table = pd_read_json(csv_table)
csv_table = csv_table[csv_table["i"].isin(gdf.index)
& csv_table["j"].isin(gdf.index)]
geoms_loc = gdf.geometry.loc
ft_template_start = \
'''{"type":"Feature","geometry":{"type":"LineString","coordinates":['''
geojson_features = []
for n, id_i, id_j, fij in csv_table[[field_i, field_j,
field_fij]].itertuples():
# pt1, pt2 = \
# list(geoms_loc[id_i].coords)[0], list(geoms_loc[id_j].coords)[0]
pts = \
list(geoms_loc[id_i].coords)[0] + list(geoms_loc[id_j].coords)[0]
geojson_features.append(''.join([
ft_template_start, '''[{0},{1}],[{2},{3}]'''.format(*pts),
''']},"properties":{"''',
'''i":"{0}","j":"{1}","fij":"{2}"'''.format(id_i, id_j,
fij), '''}}'''
]))
return ''.join([
'''{"type":"FeatureCollection","crs":{"type":"name","properties":'''
'''{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"}},"features":[''',
','.join(geojson_features), ''']}'''
]).encode()
def convexQuery():
"""
Purpose: Return the smallest convex polygon that contains a set of points
Input: `BBparams`: a dictionary of the datasets to be queried, and the
top left and bottom right corners of the bounding box
Output: `convex_full_results['features']+intersections`: a list of the
feature polygon of the convex hull and the points contained by it
"""
convexFC = {'type': "FeatureCollection", 'features': []}
BBparams = loads(request.args.get("BBparams", None))
# store a feature list of points contained by the bounding box
intersections, _ = queryBBoxIntersection(BBparams)
# append to `convexFC` the polygon formed by all the points within the bbox
convexFC['features'] = [createPolygonFromPoints(intersections)]
# create a dataframe with the polygon
convexParams = GeoDataFrame.from_features(convexFC, crs="EPSG:4326")
# create a geoseries of points that form the smallest convex Polygon
# "The convex hull of a geometry is the smallest convex Polygon containing
# all the points in each geometry, unless the number of points in the geometric
# object is less than three. For two points, the convex hull collapses to a
# LineString; for 1, a Point.
# - https://geopandas.readthedocs.io/en/latest/docs/reference/api/geopandas.GeoSeries.convex_hull.html
convex_full_results = loads((convexParams.convex_hull).to_json())
# return a list of the convex hull and the intersecting points found within the bounding box
return jsonify(convex_full_results['features'] + intersections)
def generate_decision_layer_cell_seabed(session, grid):
grid_cells = session.query(models.DecisionLayerCell, models.GridCell).join(
models.GridCell).filter(models.GridCell.grid_id == grid.id).order_by(
models.GridCell.id).all()
for cell in grid_cells:
grid_box = to_shape(cell.GridCell.bounding_box)
lonmin, lonmax, latmin, latmax = grid_box.bounds
bbox_string = ",".join(map(str, [latmin, lonmin, latmax, lonmax]))
wfs_url = "http://drive.emodnet-geology.eu/geoserver/EMODnetGeology/ows?service=WFS&version=1.1.0&request=GetFeature&typeName=EMODnetGeology:seabed_substrate250k_eu_sbss250k_seabed_substrate_eu&bbox=" + bbox_string + "&outputFormat=application%2Fjson"
try:
r = requests.get(wfs_url)
wfs_geo = geojson.loads(r.content)
wfs_gdf = GeoDataFrame.from_features(wfs_geo)
if len(wfs_gdf) > 0:
if len(wfs_gdf[wfs_gdf.folk_5_substrate_class.str.contains(
'2. Sand')]) > 0:
cell.DecisionLayerCell.seabed = wfs_gdf[
wfs_gdf.folk_5_substrate_class.str.contains(
'2. Sand')].shape_area.sum() / grid_box.area * 100
else:
cell.DecisionLayerCell.seabed = 0
except ValueError:
pass
session.commit()
def calc_zonal_statistics(shp_filename, raster_file, stats=None):
"""
Calculates zonal statistics for a raster file based on a shapefile.
:param shp_filename: filename path.
:type shp_filename: str
:param raster_file: filename raster.
:type raster_file: str
:param stats: list of statistics to calculate.
:type stats: list
:return: gpd.GeoDataFrame
"""
if stats is None:
stats = ['count', 'min', 'max', 'mean', 'median', 'std']
results = zonal_stats(
shp_filename,
raster_file,
nodata=-9999,
stats=stats,
all_touched=True,
geojson_out=True,
epsg=4326
)
gdf_stats = GeoDataFrame.from_features(results)
return gdf_stats
def make_geojson_links(ref_layer_geojson, csv_table, field_i, field_j,
field_fij, join_field):
gdf = GeoDataFrame.from_features(ref_layer_geojson["features"])
gdf.loc[:, join_field] = gdf.loc[:, join_field].astype(str)
gdf.set_index(join_field, inplace=True, drop=False)
gdf.geometry = _compute_centroids(gdf.geometry)
table = pd_read_json(csv_table)
table.loc[:, (field_i, field_j)] = \
table.loc[:, (field_i, field_j)].astype(str)
table = \
table[table[field_i].isin(gdf.index) & table[field_j].isin(gdf.index)]
geoms_loc = gdf.geometry.loc
ft_template_start = \
'''{"type":"Feature","geometry":{"type":"LineString","coordinates":['''
geojson_features = []
for n, id_i, id_j, fij in table[[field_i, field_j,
field_fij]].itertuples():
pts = \
list(geoms_loc[id_i].coords)[0] + list(geoms_loc[id_j].coords)[0]
geojson_features.append(''.join([
ft_template_start, '''[{0},{1}],[{2},{3}]'''.format(*pts),
''']},"properties":{"''',
'''{0}":"{1}","{2}":"{3}","{4}":"{5}"'''.format(
field_i, id_i, field_j, id_j, field_fij, fij), '''}}'''
]))
return ''.join([
'''{"type":"FeatureCollection","features":[''',
','.join(geojson_features), ''']}'''
]).encode()
def get_example_aoi(
self,
location: str = "Berlin",
as_dataframe: bool = False) -> Union[dict, GeoDataFrame]:
"""
Gets predefined, small, rectangular example aoi for the selected location.
Args:
location: Location, one of Berlin, Washington.
as_dataframe: Returns a dataframe instead of dict FeatureColletions
(default).
Returns:
Feature collection json with the selected aoi.
"""
logger.info(f"Getting small example aoi in location '{location}'.")
if location == "Berlin":
example_aoi = self.read_vector_file(
f"{str(Path(__file__).resolve().parent)}/data/aoi_berlin.geojson"
)
elif location == "Washington":
example_aoi = self.read_vector_file(
f"{str(Path(__file__).resolve().parent)}/data/aoi_washington.geojson"
)
else:
raise ValueError(
"Please select one of 'Berlin' or 'Washington' as the location!"
)
if as_dataframe:
df = GeoDataFrame.from_features(example_aoi, crs=4326)
return df
else:
return example_aoi
def read_file(filename, bbox=None, **kwargs):
"""
Returns a GeoDataFrame from a file or URL.
Parameters
----------
filename: str
Either the absolute or relative path to the file or URL to
be opened.
bbox : tuple | GeoDataFrame or GeoSeries, default None
Filter features by given bounding box, GeoSeries, or GeoDataFrame.
CRS mis-matches are resolved if given a GeoSeries or GeoDataFrame.
**kwargs:
Keyword args to be passed to the `open` or `BytesCollection` method
in the fiona library when opening the file. For more information on
possible keywords, type:
``import fiona; help(fiona.open)``
Examples
--------
>>> df = geopandas.read_file("nybb.shp")
Returns
-------
geodataframe : GeoDataFrame
"""
if _is_url(filename):
req = _urlopen(filename)
path_or_bytes = req.read()
reader = fiona.BytesCollection
else:
path_or_bytes = filename
reader = fiona.open
with fiona_env():
with reader(path_or_bytes, **kwargs) as features:
# In a future Fiona release the crs attribute of features will
# no longer be a dict. The following code will be both forward
# and backward compatible.
if hasattr(features.crs, "to_dict"):
crs = features.crs.to_dict()
else:
crs = features.crs
if bbox is not None:
if isinstance(bbox, GeoDataFrame) or isinstance(
bbox, GeoSeries):
bbox = tuple(bbox.to_crs(crs).total_bounds)
assert len(bbox) == 4
f_filt = features.filter(bbox=bbox)
else:
f_filt = features
columns = list(
features.meta["schema"]["properties"]) + ["geometry"]
gdf = GeoDataFrame.from_features(f_filt, crs=crs, columns=columns)
return gdf
def compute(self):
self.output.create_output_dir(self.output.uri)
features = gdal_zonalstats(
self.inputs[1].read(format=formats.JSON,
epsg=self.inputs[0].get_epsg()),
self.inputs[0].read())
self.output.data = GeoDataFrame.from_features(features)
self.output.write()
def geojson_2_geodataframe_features(geo):
with BytesCollection(bytes(geojson.dumps(geo),
encoding='utf8')) as features:
crs = features.crs
columns = list(features.meta["schema"]["properties"]) + ["geometry"]
gdf = GeoDataFrame.from_features(features, crs=crs)
gdf = gdf[columns]
return gdf
def compute(self):
self.output.create_output_dir(self.output.uri)
features = gdal_zonalstats(
self.inputs[1].read(format=formats.JSON,
epsg=self.inputs[0].get_epsg()),
self.inputs[0].read())
self.output.data = GeoDataFrame.from_features(features)
self.output.write()
def load_footprints(footprint_file_name: str) -> GeoDataFrame:
with fiona.open(footprint_file_name) as features:
properties_to_keep = [] # None of the properties here are useful
df = GeoDataFrame.from_features(
features_slimmed(features, properties_to_keep))
df.crs = features.crs
return remove_invalid_geometries(df.to_crs(epsg=4326))
def vectorize(src=None, image=None, transform=None, crs=None):
"""
Raster-to-Vector conversion.
Performs a raster-to-vector conversion of a classified image.
Parameters
----------
src: Rasterio datasource
A rasterio-style datasource created using:
with rasterio.open('path') as src.
The datasource referred to must be a classified image.
This parameter is optional. If it is not provided then the
image and the transform must be provided.
image: numpy.array
A signle band of (classified, ideally) image data where the pixel
values are integers. Shape is (1, rows, columns). This parameter is
optional.
transform: rasterio.transform
A raster transform used to convert row/column values to geographic
coordinates. This parameter is optional.
crs: rasterio.crs
A proj4 string representing the coordinate reference system.
This parameter is optional.
Returns
-------
GeoDataFrame
A vector version of the classified raster.
"""
if src is not None:
img = src.read(1, masked=True)
transform = src.transform
crs = src.crs.to_proj4()
else:
img = image[0].astype(np.int32)
# shps = features.shapes(img, connectivity=8, transform=transform)
shps = features.shapes(img, transform=transform)
records = []
for id, shp in enumerate(shps):
if shp[1] != 0:
item = {
'geometry': shp[0],
'id': id + 1,
'properties': OrderedDict([('dn', np.int(shp[1]))]),
'type': 'Feature'
}
records.append(item)
vec = GeoDataFrame.from_features(records)
vec.crs = crs
return vec
def json_obj_to_geodf(json_obj, epsg):
"""
Json Object to GeoDataFrame
"""
from geopandas import GeoDataFrame
return GeoDataFrame.from_features(json_obj['features'],
'EPSG:{}'.format(str(epsg)))
def test_from_features(self):
nybb_filename = geopandas.datasets.get_path('nybb')
with fiona.open(nybb_filename) as f:
features = list(f)
crs = f.crs
df = GeoDataFrame.from_features(features, crs=crs)
validate_boro_df(df, case_sensitive=True)
assert df.crs == crs
def json_obj_to_geodf(json_obj, epsg):
"""
Json Object to GeoDataFrame
"""
from geopandas import GeoDataFrame
return GeoDataFrame.from_features(json_obj['features'],
{'init': 'epsg:{}'.format(epsg)})
def read_file(filename, bbox=None, **kwargs):
"""
Returns a GeoDataFrame from a file or URL.
Parameters
----------
filename: str
Either the absolute or relative path to the file or URL to
be opened.
bbox : tuple | GeoDataFrame or GeoSeries, default None
Filter features by given bounding box, GeoSeries, or GeoDataFrame.
CRS mis-matches are resolved if given a GeoSeries or GeoDataFrame.
**kwargs:
Keyword args to be passed to the `open` or `BytesCollection` method
in the fiona library when opening the file. For more information on
possible keywords, type:
``import fiona; help(fiona.open)``
Examples
--------
>>> df = geopandas.read_file("nybb.shp")
Returns
-------
geodataframe : GeoDataFrame
"""
if _is_url(filename):
req = _urlopen(filename)
path_or_bytes = req.read()
reader = fiona.BytesCollection
else:
path_or_bytes = filename
reader = fiona.open
with fiona_env():
with reader(path_or_bytes, **kwargs) as features:
# In a future Fiona release the crs attribute of features will
# no longer be a dict. The following code will be both forward
# and backward compatible.
if hasattr(features.crs, 'to_dict'):
crs = features.crs.to_dict()
else:
crs = features.crs
if bbox is not None:
if isinstance(bbox, GeoDataFrame) or isinstance(bbox, GeoSeries):
bbox = tuple(bbox.to_crs(crs).total_bounds)
assert len(bbox) == 4
f_filt = features.filter(bbox=bbox)
else:
f_filt = features
columns = list(features.meta["schema"]["properties"]) + ["geometry"]
gdf = GeoDataFrame.from_features(f_filt, crs=crs, columns=columns)
return gdf
def add_message():
data = request.json
gdf_points: GeoDataFrame = GeoDataFrame.from_features(data["features"])
polygon: Polygon = gdf_points['geometry'].agg(lambda point: Polygon(point.tolist()))
gdf_poly = GeoDataFrame([{'id': 1, 'geometry': polygon}])
print(gdf_poly.to_json())
return gdf_poly.to_json()
def test_from_features(self):
nybb_filename = geopandas.datasets.get_path("nybb")
with fiona.open(nybb_filename) as f:
features = list(f)
crs = f.crs
df = GeoDataFrame.from_features(features, crs=crs)
validate_boro_df(df, case_sensitive=True)
assert df.crs == crs
def search(self,
search_parameters: dict,
as_dataframe: bool = True) -> Union[GeoDataFrame, dict]:
"""
Searches the catalog for the the search parameters and returns the metadata of
the matching scenes.
Args:
search_parameters: The catalog search parameters, see example.
as_dataframe: return type, GeoDataFrame if True (default), FeatureCollection if False.
Returns:
The search results as a GeoDataFrame, optionally as json dict.
Example:
```python
search_parameters={
"datetime": "2019-01-01T00:00:00Z/2019-01-15T23:59:59Z",
"intersects": {
"type": "Polygon",
"coordinates": [[[13.32113746,52.73971768],[13.15981158,52.2092959],
[13.62204483,52.15632025],[13.78859517,52.68655119],[13.32113746,
52.73971768]]]},
"limit": 10,
"sortby": [{"field" : "properties.acquisitionDate", "direction" : "asc"}]
}
```
"""
logger.info(
f"Searching catalog with search_parameters: {search_parameters}")
url = f"{self.auth._endpoint()}/catalog/stac/search"
response_json: dict = self.auth._request("POST", url,
search_parameters)
logger.info(f"{len(response_json['features'])} results returned.")
dst_crs = "EPSG:4326"
df = GeoDataFrame.from_features(response_json, crs=dst_crs)
if df.empty:
if as_dataframe:
return df
else:
return df.__geo_interface__
# Filter to actual geometries intersecting the aoi (Sobloo search uses a rectangular
# bounds geometry, can contain scenes that touch the aoi bbox, but not the aoi.
# So number returned images not consistent with set limit.
# TODO: Resolve on backend
geometry = search_parameters["intersects"]
poly = shape(geometry)
df = df[df.intersects(poly)]
df = df.reset_index(drop=True)
df.crs = dst_crs # apply resets the crs
if as_dataframe:
return df
else:
return df.__geo_interface__
def test_from_features(self):
nybb_filename, nybb_zip_path = download_nybb()
with fiona.open(nybb_zip_path, vfs='zip://' + nybb_filename) as f:
features = list(f)
crs = f.crs
df = GeoDataFrame.from_features(features, crs=crs)
df.rename(columns=lambda x: x.lower(), inplace=True)
validate_boro_df(self, df)
self.assert_(df.crs == crs)
def test_from_features(self):
nybb_filename = download_nybb()
with fiona.open("/nybb_14a_av/nybb.shp", vfs="zip://" + nybb_filename) as f:
features = list(f)
crs = f.crs
df = GeoDataFrame.from_features(features, crs=crs)
df.rename(columns=lambda x: x.lower(), inplace=True)
validate_boro_df(self, df)
self.assert_(df.crs == crs)
def test_from_features(self):
nybb_filename = geopandas.datasets.get_path('nybb')
with fiona.open(nybb_filename) as f:
features = list(f)
crs = f.crs
df = GeoDataFrame.from_features(features, crs=crs)
df.rename(columns=lambda x: x.lower(), inplace=True)
validate_boro_df(self, df)
self.assert_(df.crs == crs)
def test_from_features(self):
nybb_filename = geopandas.datasets.get_path('nybb')
with fiona.open(nybb_filename) as f:
features = list(f)
crs = f.crs
df = GeoDataFrame.from_features(features, crs=crs)
df.rename(columns=lambda x: x.lower(), inplace=True)
validate_boro_df(df)
assert df.crs == crs
def network_as_dataframe(self):
try:
from geopandas import GeoDataFrame
result = GeoDataFrame.from_features(self['features'])
result['id'] = [f['id'] for f in self['features']]
return result
except Exception as e:
print('Could not create GeoDataFrame. Using regular DataFrame.')
print(str(e))
return self['features'].as_dataframe()
def load_parcels(parcel_file_name: str) -> GeoDataFrame:
with fiona.open(parcel_file_name) as features:
properties_to_keep = ['PARCELID', 'RESYRBLT']
slim_features = features_slimmed(
features, properties_to_keep) # Reduces memory usage
slim_features = (
f for f in slim_features if f['geometry'] is not None
) # Apparently there are things in here with no shape?
gdf = GeoDataFrame.from_features(slim_features)
gdf.crs = features.crs
return remove_invalid_geometries(gdf.to_crs(epsg=4326))
def load_osu_buildings(building_file_name: str, data_dir: str) -> GeoDataFrame:
# TODO find a way to download this file
with fiona.open(building_file_name) as features:
gdf = GeoDataFrame.from_features(features)
gdf.crs = features.crs
building_ages = download_osu_buildings_ages(building_file_name, data_dir)
gdf = gdf.merge(building_ages, on='BLDG_NUM')
gdf = gdf[['geometry',
'year_built']] # Remove all the columns we don't need
return remove_invalid_geometries(gdf.to_crs(epsg=4326))
def test_from_feature_collection(self):
data = {'name': ['a', 'b', 'c'],
'lat': [45, 46, 47.5],
'lon': [-120, -121.2, -122.9]}
df = pd.DataFrame(data)
geometry = [Point(xy) for xy in zip(df['lon'], df['lat'])]
gdf = GeoDataFrame(df, geometry=geometry)
# from_features returns sorted columns
expected = gdf[['geometry', 'lat', 'lon', 'name']]
# test FeatureCollection
res = GeoDataFrame.from_features(gdf.__geo_interface__)
assert_frame_equal(res, expected)
# test list of Features
res = GeoDataFrame.from_features(gdf.__geo_interface__['features'])
assert_frame_equal(res, expected)
# test __geo_interface__ attribute (a GeoDataFrame has one)
res = GeoDataFrame.from_features(gdf)
assert_frame_equal(res, expected)
def test_from_feature_collection(self):
data = {'name': ['a', 'b', 'c'],
'lat': [45, 46, 47.5],
'lon': [-120, -121.2, -122.9]}
df = pd.DataFrame(data)
geometry = [Point(xy) for xy in zip(df['lon'], df['lat'])]
gdf = GeoDataFrame(df, geometry=geometry)
# from_features returns sorted columns
expected = gdf[['geometry', 'lat', 'lon', 'name']]
# test FeatureCollection
res = GeoDataFrame.from_features(gdf.__geo_interface__)
assert_frame_equal(res, expected)
# test list of Features
res = GeoDataFrame.from_features(gdf.__geo_interface__['features'])
assert_frame_equal(res, expected)
# test __geo_interface__ attribute (a GeoDataFrame has one)
res = GeoDataFrame.from_features(gdf)
assert_frame_equal(res, expected)
def queryBBoxIntersection(BBparams):
"""
Purpose: Performs a bounding box query on all selected datasets
Input: `BBparams`: a dictionary of the datasets to be queried, and the
top left and bottom right corners of the bounding box
Output: `full_results`: a list of the feature documents of all points
contained within the bounding box
`bbox['features']`: a feature document of the bounding box
"""
# store the top, left, bottom, and right coordinate borders of the bounding box
top, left, bottom, right = \
float(BBparams["bbox"][0].split(',')[0]), \
float(BBparams["bbox"][0].split(',')[1]), \
float(BBparams["bbox"][1].split(',')[0]), \
float(BBparams["bbox"][1].split(',')[1])
# the bounding box's polygon feature
bbox = {
'type':
"FeatureCollection",
"features": [{
"type": "Feature",
"properties": {
"name": "United States"
},
"geometry": {
"type":
"Polygon",
"coordinates": [[[top, left], [top, right], [bottom, right],
[bottom, left], [top, left]]]
}
}]
}
# create a dataframe of the `bbox` feature
bbox_df = GeoDataFrame.from_features(bbox, crs="EPSG:4326")
# load the datasets into memory
process_datasets(BBparams["datasets"])
full_results = []
for dataset in dataset_map:
# only query the datasets that were checkboxed in the frontend
if BBparams['datasets'][dataset]:
# queries the dataset dataframe for all points in `bbox_df`'s bounding box
points_in_bbox, _ = bbox_df.sindex.query_bulk(
dataset_map[dataset]["dataframe"].geometry,
predicate='intersects')
# create a dataframe of all the intersecting points in dataset dataframe
matches = dataset_map[dataset]["dataframe"].iloc[points_in_bbox]
full_results += loads(matches.to_json())['features']
return full_results, bbox['features']
def dump_elbs(year=2016):
storage_key = settings['azure']['pcs_storage_key']
account = az.CloudStorageAccount(account_name='pcslive', account_key=storage_key)
blob_service = account.create_block_blob_service()
year_ids = elb_repo.get_elb_harvest_year_ids(year=2016)
if not os.path.exists('data/elbs'): os.mkdir('data/elbs')
for idx, elb_year_id in enumerate(year_ids):
print("downloading elb GIS cells. idx, yearid: ({} of {}), {}".format(idx, len(year_ids), elb_year_id))
crop = gis_repo.get_pps_crop(elb_year_id)
if not 'Corn' in crop:
print("found not-corn crop, ignoring: {}".format(crop))
continue
# use the harvest layers
elb_source_layers = [
b.name
for b in list(blob_service.list_blobs('sourcelayers', str(elb_year_id)))
if any(x in b.name for x in ['_13_', '_14_', '_15_'])]
elb_harvest_source_layer_name = elb_source_layers[0] if len(elb_source_layers) > 0 else None
if elb_harvest_source_layer_name is None:
print("ELB has no harvest layer: {}".format(elb_year_id))
continue
blob_zip = blob_service.get_blob_to_bytes('sourcelayers', elb_harvest_source_layer_name)
vsiz = '/vsimem/{}.zip'.format(uuid.uuid4().hex) # gdal/ogr requires a .zip extension
FileFromMemBuffer(vsiz, bytes(blob_zip.content))
with fiona.Collection(vsiz, vsi='zip') as f:
shp = GeoDataFrame.from_features(f, crs={'init': 'epsg:4326'})
elb_points = GeoDataFrame(shp.loc[shp['ELB_ID'] > 0])
elb_centroids = list(elb_points.centroid)
pps = gis_repo.processed_layer_shapes_by_year_id(elb_year_id)
# get pps cells that have an elb
pps_elb_cells = DataFrame(
pps.loc[pps['geometry'].apply(lambda x: any(x.intersects(c) for c in elb_centroids))])
pps_elb_cells.drop(['geometry'], inplace=True, axis=1)
# load weather record
wx = gis_repo.weather_by_year_id(elb_year_id)
pps_elb_cells = pandas.concat([
pps_elb_cells,
pandas.DataFrame([wx.values], index=pps_elb_cells.index, columns=wx.keys())], axis=1)
pps_elb_cells.to_pickle(f'data/elbs/{elb_year_id}_elb.pickle.gz', compression='gzip')
def from_dict(cls, data: dict) -> "FeatureCollection":
"""
Create a feature collection from a python dictionary that was created from
the JSON definition of the FeatureCollection
:param data: The dictionary that contains the feature collection definition
:return: A new FeatureCollection object
"""
return cls(
id=data["id"],
data=GeoDataFrame.from_features(data["data"]),
start_times=data.get("start_times"),
end_times=data.get("end_times"),
)
def test_fishnet_geojson_string_return(self):
"""
Tests fishnet generation with a GeoJSON string return
"""
self.logger.info('Fishnet with GeoJSON string return...')
geojson = FishNet(outfile=None,
outformat='GeoJSON',
bbox=[414650, 563500, 429600, 575875]).create()
self.assertFalse(geojson is None)
try:
gdf = GeoDataFrame.from_features(geojson)
self.logger.info(gdf.head(10))
except ValueError:
self.fail('Returned GeoJSON could not be read into a GeoDataFrame')
self.logger.info('Completed')
def test_from_features_unaligned_properties(self):
p1 = Point(1, 1)
f1 = {"type": "Feature", "properties": {"a": 0}, "geometry": p1.__geo_interface__}
p2 = Point(2, 2)
f2 = {"type": "Feature", "properties": {"b": 1}, "geometry": p2.__geo_interface__}
p3 = Point(3, 3)
f3 = {"type": "Feature", "properties": {"a": 2}, "geometry": p3.__geo_interface__}
df = GeoDataFrame.from_features([f1, f2, f3])
result = df[["a", "b"]]
expected = pd.DataFrame.from_dict([{"a": 0, "b": np.nan}, {"a": np.nan, "b": 1}, {"a": 2, "b": np.nan}])
assert_frame_equal(expected, result)
def test_from_feature_collection(self):
data = {
"name": ["a", "b", "c"],
"lat": [45, 46, 47.5],
"lon": [-120, -121.2, -122.9],
}
df = pd.DataFrame(data)
geometry = [Point(xy) for xy in zip(df["lon"], df["lat"])]
gdf = GeoDataFrame(df, geometry=geometry)
# from_features returns sorted columns
expected = gdf[["geometry", "lat", "lon", "name"]]
# test FeatureCollection
res = GeoDataFrame.from_features(gdf.__geo_interface__)
assert_frame_equal(res, expected)
# test list of Features
res = GeoDataFrame.from_features(gdf.__geo_interface__["features"])
assert_frame_equal(res, expected)
# test __geo_interface__ attribute (a GeoDataFrame has one)
res = GeoDataFrame.from_features(gdf)
assert_frame_equal(res, expected)
def read_file(filename, **kwargs):
"""
Returns a GeoDataFrame from a file.
*filename* is either the absolute or relative path to the file to be
opened and *kwargs* are keyword args to be passed to the method when
opening the file.
"""
bbox = kwargs.pop('bbox', None)
with fiona.open(filename, **kwargs) as f:
crs = f.crs
if bbox is not None:
assert len(bbox)==4
f_filt = f.filter(bbox=bbox)
else:
f_filt = f
gdf = GeoDataFrame.from_features(f, crs=crs)
return gdf
def read_file(filename, **kwargs):
"""
Returns a GeoDataFrame from a file.
*filename* is either the absolute or relative path to the file to be
opened and *kwargs* are keyword args to be passed to the `open` method
in the fiona library when opening the file. For more information on
possible keywords, type: ``import fiona; help(fiona.open)``
"""
bbox = kwargs.pop('bbox', None)
with fiona.open(filename, **kwargs) as f:
crs = f.crs
if bbox is not None:
assert len(bbox)==4
f_filt = f.filter(bbox=bbox)
else:
f_filt = f
gdf = GeoDataFrame.from_features(f_filt, crs=crs)
return gdf
def test_from_features_unaligned_properties(self):
p1 = Point(1,1)
f1 = {'type': 'Feature',
'properties': {'a': 0},
'geometry': p1.__geo_interface__}
p2 = Point(2,2)
f2 = {'type': 'Feature',
'properties': {'b': 1},
'geometry': p2.__geo_interface__}
p3 = Point(3,3)
f3 = {'type': 'Feature',
'properties': {'a': 2},
'geometry': p3.__geo_interface__}
df = GeoDataFrame.from_features([f1, f2, f3])
result = df[['a', 'b']]
expected = pd.DataFrame.from_dict([{'a': 0, 'b': np.nan},
{'a': np.nan, 'b': 1},
{'a': 2, 'b': np.nan}])
assert_frame_equal(expected, result)
def read_file(filename, **kwargs):
"""
Returns a GeoDataFrame from a file.
*filename* is either the absolute or relative path to the file to be
opened and *kwargs* are keyword args to be passed to the `open` method
in the fiona library when opening the file. For more information on
possible keywords, type: ``import fiona; help(fiona.open)``
"""
bbox = kwargs.pop('bbox', None)
with fiona.open(filename, **kwargs) as f:
crs = f.crs
if bbox is not None:
assert len(bbox)==4
f_filt = f.filter(bbox=bbox)
else:
f_filt = f
gdf = GeoDataFrame.from_features(f_filt, crs=crs)
# re-order with column order from metadata, with geometry last
columns = list(f.meta["schema"]["properties"]) + ["geometry"]
gdf = gdf[columns]
return gdf
def network_as_dataframe(self):
from geopandas import GeoDataFrame
result = GeoDataFrame.from_features(self['features'])
result['id'] = [f['id'] for f in self['features']]
return result
