def _parse_shapefile(name): """ Downloads the provided shapefile from GeoMAC. Returns a GeoJSON object. """ # Figure out the url domain = 'rmgsc.cr.usgs.gov' base_dir = 'outgoing/GeoMAC/current_year_fire_data/current_year_all_states' url = f'https://{domain}/{base_dir}/{name}' # Get the zipfile r = requests.get(url) buffer = io.BytesIO(bytes(r.content)) shp = fiona.BytesCollection(buffer.getvalue()) # Convert it GeoJSON and return it feature_list = [ Feature(geometry=d['geometry'], properties=d['properties']) for d in shp ] return FeatureCollection(feature_list)
def load_shpfiles(): def clip_geoid(x): return x[9:] # read shapefiles as geodataframes path_list = list_shpfiles() main_crs = None gdf_list = [] s_time = time.time() total_cts = 0 print('loading shapefiles') for path in path_list: # print(path) inf = open(path, 'rb') direct = inf.read() with fiona.BytesCollection(direct) as f: crs = f.crs ct_gdf = gpd.GeoDataFrame.from_features(f, crs=crs) # print("Shape of the ct_gdf: {}".format(ct_gdf.shape)) # print("Projection of ct_gdf: {}".format(ct_gdf.crs)) ct_gdf = ct_gdf.to_crs({'init': 'epsg:4326'}) # print("Projection of ct_gdf: {}".format(ct_gdf.crs)) if not main_crs: main_crs = ct_gdf.crs gdf_list.append(ct_gdf) total_cts += ct_gdf.shape[0] inf.close() print('loading done, takes {} seconds'.format(time.time() - s_time)) print('{} states shpfiles are loaded, {} census tracts'.format( len(gdf_list), total_cts)) # concat all geodataframes all_gdf = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True), crs=main_crs) all_gdf['GEOID'] = all_gdf['GEO_ID'].apply(clip_geoid) print("Shape of the all_gdf: {}".format(all_gdf.shape)) print("Projection of all_gdf: {}".format(all_gdf.crs)) # print(all_gdf.loc[all_gdf['GEOID'] == '47053966500']) return all_gdf, main_crs
def load_shpfiles(states, state_dict, base_path): def clip_geoid(x): return x[9:] # read shapefiles as geodataframes main_crs = None gdf_list = [] for s in states: fips = state_dict[s] path = base_path.format(fips) with open(path, 'rb') as inf: direct = inf.read() with fiona.BytesCollection(direct) as f: crs = f.crs ct_gdf = gpd.GeoDataFrame.from_features(f, crs=crs) ct_gdf = ct_gdf.to_crs({'init': 'epsg:4326'}) if not main_crs: main_crs = ct_gdf.crs gdf_list.append(ct_gdf) # concat all geodataframes all_gdf = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True), crs=main_crs) all_gdf['GEOID'] = all_gdf['GEO_ID'].apply(clip_geoid) return all_gdf, main_crs
def test_zipped_bytes_collection(bytes_coutwildrnp_zip): """Open a zipped stream of bytes as a collection""" with fiona.BytesCollection(bytes_coutwildrnp_zip) as col: assert col.name == 'coutwildrnp' assert len(col) == 67
def test_construct_with_str(self, path_coutwildrnp_json): with open(path_coutwildrnp_json) as src: strbuf = src.read() with pytest.raises(ValueError): fiona.BytesCollection(strbuf)
def bytes_collection_object(self, path_coutwildrnp_json): with open(path_coutwildrnp_json) as src: bytesbuf = src.read().encode('utf-8') self.c = fiona.BytesCollection(bytesbuf) yield self.c.close()
#import contextily as ctx #import matplotlib.pyplot as plt #import mplleaflet #from mpl_toolkits.basemap import Basemap #import folium #from folium import FeatureGroup, LayerControl, Map, Marker #from folium.plugins import HeatMap #Get dataset with polygons, so we van calculate areas link = 'https://services5.arcgis.com/QYf5PkPqzJKVzrmF/arcgis/rest/services/Rohingya_Refugee_Camps_Sites_Outline_May_18/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryPolygon&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&returnCentroid=false&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnDistinctValues=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pgeojson&token=' request = requests.get(link) b = bytes(request.content) with fiona.BytesCollection(b) as f: crs = f.crs data_polygons = gpd.GeoDataFrame.from_features(f, crs=crs) #Give both datasets the same name data_polygons = data_polygons.rename(columns={'New_Camp_N': 'New_Camp_Name'}) print(data_polygons) #Get dataset with populations link = 'https://services5.arcgis.com/QYf5PkPqzJKVzrmF/arcgis/rest/services/Latest_Location_Masterlist_June_2018/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnDistinctValues=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pgeojson&token=' request = requests.get(link) b = bytes(request.content) with fiona.BytesCollection(b) as f: crs = f.crs data_pop = gpd.GeoDataFrame.from_features(f, crs=crs)
def query_ons_records(base_query, time_between_queries=1, save_path=None, overwrite=False): if save_path and os.path.exists(save_path) and not overwrite: return gpd.read_file(save_path) offset_param = "&resultOffset={}" count_param = "&returnCountOnly=true" r = requests.get(base_query + count_param) j = r.json() n_records_to_query = j["count"] if n_records_to_query > 0: print("This query returns", n_records_to_query, "records.") else: raise ValueError("Input query returns no records.") n_queried_records = 0 all_records = None while n_queried_records < n_records_to_query: print("PROGRESS:", n_queried_records, "out of", n_records_to_query, "records") start_time = time.time() print("Querying... ", end="") try: r = requests.get(base_query + offset_param.format(n_queried_records)) except requests.exceptions.Timeout: print("timeout, retrying...") for i in range(10): print("attempt", i + 1) try: r = requests.get(base_query + offset_param.format(n_queried_records)) break except requests.exceptions.Timeout: r = None continue if not r: raise requests.exceptions.Timeout("FAILED - timeout.") j = r.json() n_new_records = len(j["features"]) n_queried_records += n_new_records print("Got", n_new_records, "records.") if n_new_records > 0: b = bytes(r.content) with fiona.BytesCollection(b) as f: crs = f.crs new_records = gpd.GeoDataFrame.from_features(f, crs=crs) if all_records is None: all_records = new_records.copy(deep=True) else: all_records = all_records.append(new_records) if "exceededTransferLimit" in j.keys( ) and j["exceededTransferLimit"] is True: end_time = time.time() if end_time - start_time < time_between_queries: time.sleep(time_between_queries + start_time - end_time) continue else: print("No more records to query.") break if save_path: os.makedirs(os.path.dirname(save_path), exist_ok=True) all_records.to_file(save_path) return all_records
def setUp(self): with open('tests/data/coutwildrnp.json') as src: bytesbuf = src.read().encode('utf-8') self.c = fiona.BytesCollection(bytesbuf)
def setUp(self): with open(self.path_coutwildrnp_json) as src: bytesbuf = src.read().encode('utf-8') self.c = fiona.BytesCollection(bytesbuf)