def _parse_shapefile(name):
"""
Downloads the provided shapefile from GeoMAC.
Returns a GeoJSON object.
"""
# Figure out the url
domain = 'rmgsc.cr.usgs.gov'
base_dir = 'outgoing/GeoMAC/current_year_fire_data/current_year_all_states'
url = f'https://{domain}/{base_dir}/{name}'
# Get the zipfile
r = requests.get(url)
buffer = io.BytesIO(bytes(r.content))
shp = fiona.BytesCollection(buffer.getvalue())
# Convert it GeoJSON and return it
feature_list = [
Feature(geometry=d['geometry'], properties=d['properties'])
for d in shp
]
return FeatureCollection(feature_list)
def load_shpfiles():
def clip_geoid(x):
return x[9:]
# read shapefiles as geodataframes
path_list = list_shpfiles()
main_crs = None
gdf_list = []
s_time = time.time()
total_cts = 0
print('loading shapefiles')
for path in path_list:
# print(path)
inf = open(path, 'rb')
direct = inf.read()
with fiona.BytesCollection(direct) as f:
crs = f.crs
ct_gdf = gpd.GeoDataFrame.from_features(f, crs=crs)
# print("Shape of the ct_gdf: {}".format(ct_gdf.shape))
# print("Projection of ct_gdf: {}".format(ct_gdf.crs))
ct_gdf = ct_gdf.to_crs({'init': 'epsg:4326'})
# print("Projection of ct_gdf: {}".format(ct_gdf.crs))
if not main_crs:
main_crs = ct_gdf.crs
gdf_list.append(ct_gdf)
total_cts += ct_gdf.shape[0]
inf.close()
print('loading done, takes {} seconds'.format(time.time() - s_time))
print('{} states shpfiles are loaded, {} census tracts'.format(
len(gdf_list), total_cts))
# concat all geodataframes
all_gdf = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True),
crs=main_crs)
all_gdf['GEOID'] = all_gdf['GEO_ID'].apply(clip_geoid)
print("Shape of the all_gdf: {}".format(all_gdf.shape))
print("Projection of all_gdf: {}".format(all_gdf.crs))
# print(all_gdf.loc[all_gdf['GEOID'] == '47053966500'])
return all_gdf, main_crs
def load_shpfiles(states, state_dict, base_path):
def clip_geoid(x):
return x[9:]
# read shapefiles as geodataframes
main_crs = None
gdf_list = []
for s in states:
fips = state_dict[s]
path = base_path.format(fips)
with open(path, 'rb') as inf:
direct = inf.read()
with fiona.BytesCollection(direct) as f:
crs = f.crs
ct_gdf = gpd.GeoDataFrame.from_features(f, crs=crs)
ct_gdf = ct_gdf.to_crs({'init': 'epsg:4326'})
if not main_crs:
main_crs = ct_gdf.crs
gdf_list.append(ct_gdf)
# concat all geodataframes
all_gdf = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True),
crs=main_crs)
all_gdf['GEOID'] = all_gdf['GEO_ID'].apply(clip_geoid)
return all_gdf, main_crs
def test_zipped_bytes_collection(bytes_coutwildrnp_zip):
"""Open a zipped stream of bytes as a collection"""
with fiona.BytesCollection(bytes_coutwildrnp_zip) as col:
assert col.name == 'coutwildrnp'
assert len(col) == 67
def test_construct_with_str(self, path_coutwildrnp_json):
with open(path_coutwildrnp_json) as src:
strbuf = src.read()
with pytest.raises(ValueError):
fiona.BytesCollection(strbuf)
def bytes_collection_object(self, path_coutwildrnp_json):
with open(path_coutwildrnp_json) as src:
bytesbuf = src.read().encode('utf-8')
self.c = fiona.BytesCollection(bytesbuf)
yield
self.c.close()
#import contextily as ctx
#import matplotlib.pyplot as plt
#import mplleaflet
#from mpl_toolkits.basemap import Basemap
#import folium
#from folium import FeatureGroup, LayerControl, Map, Marker
#from folium.plugins import HeatMap
#Get dataset with polygons, so we van calculate areas
link = 'https://services5.arcgis.com/QYf5PkPqzJKVzrmF/arcgis/rest/services/Rohingya_Refugee_Camps_Sites_Outline_May_18/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryPolygon&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&returnCentroid=false&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnDistinctValues=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pgeojson&token='
request = requests.get(link)
b = bytes(request.content)
with fiona.BytesCollection(b) as f:
crs = f.crs
data_polygons = gpd.GeoDataFrame.from_features(f, crs=crs)
#Give both datasets the same name
data_polygons = data_polygons.rename(columns={'New_Camp_N': 'New_Camp_Name'})
print(data_polygons)
#Get dataset with populations
link = 'https://services5.arcgis.com/QYf5PkPqzJKVzrmF/arcgis/rest/services/Latest_Location_Masterlist_June_2018/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnDistinctValues=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pgeojson&token='
request = requests.get(link)
b = bytes(request.content)
with fiona.BytesCollection(b) as f:
crs = f.crs
data_pop = gpd.GeoDataFrame.from_features(f, crs=crs)
def query_ons_records(base_query,
time_between_queries=1,
save_path=None,
overwrite=False):
if save_path and os.path.exists(save_path) and not overwrite:
return gpd.read_file(save_path)
offset_param = "&resultOffset={}"
count_param = "&returnCountOnly=true"
r = requests.get(base_query + count_param)
j = r.json()
n_records_to_query = j["count"]
if n_records_to_query > 0:
print("This query returns", n_records_to_query, "records.")
else:
raise ValueError("Input query returns no records.")
n_queried_records = 0
all_records = None
while n_queried_records < n_records_to_query:
print("PROGRESS:", n_queried_records, "out of", n_records_to_query,
"records")
start_time = time.time()
print("Querying... ", end="")
try:
r = requests.get(base_query +
offset_param.format(n_queried_records))
except requests.exceptions.Timeout:
print("timeout, retrying...")
for i in range(10):
print("attempt", i + 1)
try:
r = requests.get(base_query +
offset_param.format(n_queried_records))
break
except requests.exceptions.Timeout:
r = None
continue
if not r:
raise requests.exceptions.Timeout("FAILED - timeout.")
j = r.json()
n_new_records = len(j["features"])
n_queried_records += n_new_records
print("Got", n_new_records, "records.")
if n_new_records > 0:
b = bytes(r.content)
with fiona.BytesCollection(b) as f:
crs = f.crs
new_records = gpd.GeoDataFrame.from_features(f, crs=crs)
if all_records is None:
all_records = new_records.copy(deep=True)
else:
all_records = all_records.append(new_records)
if "exceededTransferLimit" in j.keys(
) and j["exceededTransferLimit"] is True:
end_time = time.time()
if end_time - start_time < time_between_queries:
time.sleep(time_between_queries + start_time - end_time)
continue
else:
print("No more records to query.")
break
if save_path:
os.makedirs(os.path.dirname(save_path), exist_ok=True)
all_records.to_file(save_path)
return all_records
def setUp(self):
with open('tests/data/coutwildrnp.json') as src:
bytesbuf = src.read().encode('utf-8')
self.c = fiona.BytesCollection(bytesbuf)
def setUp(self):
with open(self.path_coutwildrnp_json) as src:
bytesbuf = src.read().encode('utf-8')
self.c = fiona.BytesCollection(bytesbuf)
